feat: better logs view

.env.example

@@ -2,4 +2,6 @@ NAV_BASE_URL=your_navidrome_server_url
 NAV_USER=your_navidrome_username
 NAV_PASS=your_navidrome_password
 SPOTIFY_CLIENT_ID=your_spotify_client_id
-SPOTIFY_CLIENT_SECRET=your_spotify_client_secret
+SPOTIFY_CLIENT_SECRET=your_spotify_client_secret
+WEB_API_KEY=changeme
+ENABLE_WEB_API_KEY=True

.gitignore

@@ -1,3 +1,4 @@
-logs/*.log
+data/
 .env
 docker-compose.yml
+__pycache__

Dockerfile

@@ -10,5 +10,8 @@ COPY . .
 # Install any needed packages specified in requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Use an entrypoint script
-ENTRYPOINT ["python", "./sptnr.py"]
+# Expose port 5000 for the Flask app
+EXPOSE 5000
+
+# Set the entrypoint to Python
+ENTRYPOINT ["python", "sptnr.py"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Kevin Restaino
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

VERSION

@@ -1 +1 @@
-1.0.0
+2.0.0-alpha

build_and_push.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Ensure the script stops if there is an error
+set -e
+
+# Read version from the VERSION file
+VERSION=$(cat VERSION)
+
+# Set up the builder instance (only needs to be done once, so you can comment this out after the first run)
+# docker buildx create --name mybuilder --use
+# docker buildx inspect mybuilder --bootstrap
+
+# Build and push the Docker image for both arm64 and amd64 platforms with the version tag
+docker buildx build --platform linux/arm64,linux/amd64 -t krestaino/sptnr:$VERSION . --push
+
+# Build and push the 'latest' tag as well
+docker buildx build --platform linux/arm64,linux/amd64 -t krestaino/sptnr:latest . --push
+
+echo "Docker images tagged and pushed: $VERSION and latest"

docker-compose.yml.example

@@ -9,11 +9,21 @@ services:
     # Uncomment the next line to build the Docker image locally
     # build: .
 
+    # Uncomment the next line to run the script
+    # entrypoint: ["python", "sptnr.py"]
+    
+    # Uncomment the next line to start the web server
+    # entrypoint: ["gunicorn", "-b", "0.0.0.0:5000", "server:sptnr"]
+
     environment:
       - NAV_BASE_URL=your_navidrome_server_url
       - NAV_USER=your_navidrome_username
       - NAV_PASS=your_navidrome_password
       - SPOTIFY_CLIENT_ID=your_spotify_client_id
       - SPOTIFY_CLIENT_SECRET=your_spotify_client_secret
+      - WEB_API_KEY=changeme
+      - ENABLE_WEB_API_KEY=False
     volumes:
-      - ./logs:/usr/src/app/logs
+      - ./data:/usr/src/app/data
+    ports:
+      - "3333:5000"

requirements.txt

@@ -1,4 +1,7 @@
 requests==2.31.0
 python-dotenv==1.0.0
 colorama==0.4.6
-tqdm==4.66.1
+tqdm==4.66.1
+gunicorn==20.1.0
+Flask==2.1.2
+Werkzeug==2.0.3

server.py

@@ -0,0 +1,133 @@
+from flask import Flask, request, jsonify
+import subprocess
+import threading
+import os
+from dotenv import load_dotenv
+import functools
+import re
+import datetime
+
+load_dotenv()
+
+sptnr = Flask(__name__)
+WEB_API_KEY = os.getenv("WEB_API_KEY")
+ENABLE_WEB_API_KEY = os.getenv("ENABLE_WEB_API_KEY", "True") == "True"
+LOG_DIR = "data/logs"
+
+
+def api_key_required(f):
+    @functools.wraps(f)
+    def decorated_function(*args, **kwargs):
+        if ENABLE_WEB_API_KEY and request.args.get("api_key") != WEB_API_KEY:
+            return jsonify({"error": "Unauthorized"}), 401
+        return f(*args, **kwargs)
+
+    return decorated_function
+
+
+def run_script(cmd):
+    subprocess.run(cmd)
+
+
+@sptnr.route("/process", methods=["GET", "POST"])
+@api_key_required
+def process_request():
+    cmd = ["python3", "sptnr.py"]
+    if request.args.get("preview", "") == "true":
+        cmd.append("--preview")
+    if request.args.get("force", "") == "true":
+        cmd.append("--force")
+    artist_ids = request.args.getlist("artist")
+    for artist_id in artist_ids:
+        cmd.extend(["--artist", artist_id])
+    album_ids = request.args.getlist("album")
+    for album_id in album_ids:
+        cmd.extend(["--album", album_id])
+    start = request.args.get("start")
+    if start:
+        cmd.extend(["--start", start])
+    limit = request.args.get("limit")
+    if limit:
+        cmd.extend(["--limit", limit])
+
+    thread = threading.Thread(target=run_script, args=(cmd,))
+    thread.start()
+
+    return jsonify({"message": "Processing started"})
+
+
+@sptnr.route("/logs")
+@api_key_required
+def list_logs():
+    try:
+        logs = os.listdir(LOG_DIR)
+        logs = [log for log in logs if log.endswith(".log")]  # Filter log files
+        full_paths = [os.path.join(LOG_DIR, log) for log in logs]
+        logs_sorted = sorted(full_paths, key=os.path.getmtime, reverse=True)
+
+        table_rows = []
+        for log_path in logs_sorted:
+            with open(log_path, "r") as file:
+                last_line = file.readlines()[-1]
+                tracks, found, not_found, match, time = parse_log_data(last_line)
+
+            log_name = os.path.basename(log_path)
+            timestamp = int(log_name.split("_")[1].split(".")[0])
+            log_datetime = datetime.datetime.fromtimestamp(timestamp)
+            formatted_datetime = log_datetime.strftime("%Y-%m-%d %H:%M:%S")
+
+            table_rows.append(
+                f"<tr><td><a href='/logs/{log_name}'>{log_name}</a></td><td>{formatted_datetime}</td><td>{tracks}</td><td>{found}</td><td>{not_found}</td><td>{match}</td><td>{time}</td></tr>"
+            )
+
+        table_html = f"<table border='1'><tr><th>Log File</th><th>Date & Time</th><th>Tracks</th><th>Found</th><th>Not Found</th><th>Match</th><th>Time</th></tr>{''.join(table_rows)}</table>"
+        return table_html
+    except Exception as e:
+        return f"An error occurred: {e}", 500
+
+
+def parse_log_data(line):
+    # Regular expressions to extract the required data
+    tracks_pattern = r"Tracks: (\d+)"
+    found_pattern = r"Found: (\d+)"
+    not_found_pattern = r"Not Found: (\d+)"
+    match_pattern = r"Match: ([\d.]+%)"
+    time_pattern = r"Time: ([\ds]+)"
+
+    # Extracting data using regular expressions
+    tracks = re.search(tracks_pattern, line)
+    found = re.search(found_pattern, line)
+    not_found = re.search(not_found_pattern, line)
+    match = re.search(match_pattern, line)
+    time = re.search(time_pattern, line)
+
+    # Getting the values or "N/A" if not found
+    tracks = tracks.group(1) if tracks else "N/A"
+    found = found.group(1) if found else "N/A"
+    not_found = not_found.group(1) if not_found else "N/A"
+    match = match.group(1) if match else "N/A"
+    time = time.group(1) if time else "N/A"
+
+    return tracks, found, not_found, match, time
+
+
+@sptnr.route("/logs/<filename>")
+@api_key_required
+def view_log(filename):
+    try:
+        full_path = os.path.join(LOG_DIR, filename)
+        if not os.path.exists(full_path) or not os.path.isfile(full_path):
+            return "File not found", 404
+
+        with open(full_path, "r") as file:
+            content = file.read()
+
+        # Convert content to HTML-friendly format
+        content = content.replace("\n", "<br>")
+        return f"<pre>{content}</pre>"
+    except Exception as e:
+        return f"An error occurred: {e}", 500
+
+
+if __name__ == "__main__":
+    sptnr.run(debug=False, host="0.0.0.0", port=3333)

sptnr.py

@@ -1,4 +1,4 @@
-with open('VERSION', 'r') as file:
+with open("VERSION", "r") as file:
     __version__ = file.read().strip()
 
 import argparse
@@ -30,28 +30,7 @@ NAV_PASS = os.getenv("NAV_PASS")
 SPOTIFY_CLIENT_ID = os.getenv("SPOTIFY_CLIENT_ID")
 SPOTIFY_CLIENT_SECRET = os.getenv("SPOTIFY_CLIENT_SECRET")
 
-# Setup logs
-LOG_DIR = "logs"
-if not os.path.exists(LOG_DIR):
-    os.makedirs(LOG_DIR)
-
-LOGFILE = os.path.join(LOG_DIR, f"spotify-popularity_{int(time.time())}.log")
-
-# Auth
-HEX_ENCODED_PASS = NAV_PASS.encode().hex()
-TOKEN_AUTH = base64.b64encode(
-    f"{SPOTIFY_CLIENT_ID}:{SPOTIFY_CLIENT_SECRET}".encode()
-).decode()
-TOKEN_URL = "https://accounts.spotify.com/api/token"
-response = requests.post(
-    TOKEN_URL,
-    headers={"Authorization": f"Basic {TOKEN_AUTH}"},
-    data={"grant_type": "client_credentials"},
-)
-SPOTIFY_TOKEN = json.loads(response.text)["access_token"]
-
-init(autoreset=True)
-
+# Colors
 LIGHT_PURPLE = Fore.MAGENTA + Style.BRIGHT
 LIGHT_GREEN = Fore.GREEN + Style.BRIGHT
 LIGHT_RED = Fore.RED + Style.BRIGHT
@@ -61,22 +40,14 @@ LIGHT_YELLOW = Fore.YELLOW + Style.BRIGHT
 BOLD = Style.BRIGHT
 RESET = Style.RESET_ALL
 
-# Default flags
-PREVIEW = 0
-START = 0
-LIMIT = 0
-ARTIST_IDs = []
-ALBUM_IDs = []
+# Setup logs
+LOG_DIR = "data/logs"
+if not os.path.exists(LOG_DIR):
+    os.makedirs(LOG_DIR)
 
-# Variables
-ARTISTS_PROCESSED = 0
-TOTAL_TRACKS = 0
-FOUND_AND_UPDATED = 0
-NOT_FOUND = 0
-UNMATCHED_TRACKS = []
+LOGFILE = os.path.join(LOG_DIR, f"sptnr_{int(time.time())}.log")
 
 
-# Setup logging
 class NoColorFormatter(logging.Formatter):
     ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
 
@@ -95,6 +66,47 @@ file_handler = logging.FileHandler(LOGFILE, "a")
 file_handler.setFormatter(NoColorFormatter("[%(asctime)s] %(message)s"))
 logging.getLogger().addHandler(file_handler)
 
+# Auth
+HEX_ENCODED_PASS = NAV_PASS.encode().hex()
+TOKEN_AUTH = base64.b64encode(
+    f"{SPOTIFY_CLIENT_ID}:{SPOTIFY_CLIENT_SECRET}".encode()
+).decode()
+TOKEN_URL = "https://accounts.spotify.com/api/token"
+response = requests.post(
+    TOKEN_URL,
+    headers={"Authorization": f"Basic {TOKEN_AUTH}"},
+    data={"grant_type": "client_credentials"},
+)
+
+if response.status_code != 200:
+    error_info = response.json()  # Assuming the error response is in JSON format
+    error_description = error_info.get("error_description", "Unknown error")
+    logging.error(
+        f"{LIGHT_RED}Spotify Authentication Error: {error_description}{RESET}"
+    )
+    sys.exit(1)
+
+SPOTIFY_TOKEN = response.json()["access_token"]
+
+init(autoreset=True)
+
+# Default flags
+PREVIEW = 0
+START = 0
+LIMIT = 0
+ARTIST_IDs = []
+ALBUM_IDs = []
+
+# Variables
+ARTISTS_PROCESSED = 0
+TOTAL_TRACKS = 0
+FOUND_AND_UPDATED = 0
+NOT_FOUND = 0
+UNMATCHED_TRACKS = []
+PROCESSED_ALBUMS_FILE = "data/processed_albums.txt"
+
+processed_albums = set()
+
 # Parse arguments
 description_text = "process command-line flags for sync"
 parser = argparse.ArgumentParser()
@@ -134,6 +146,18 @@ parser.add_argument(
     help="limit to processing [NUM] artists from the start index",
 )
 
+parser.add_argument(
+    "-v", "--version", action="version", version=f"%(prog)s {__version__}"
+)
+
+parser.add_argument(
+    "-f",
+    "--force",
+    action="store_true",
+    help="force processing of all albums, even if they were processed previously",
+)
+
+
 args = parser.parse_args()
 
 ARTIST_IDs = args.artist if args.artist else []
@@ -141,6 +165,8 @@ ALBUM_IDs = args.album if args.album else []
 START = args.start
 LIMIT = args.limit
 
+logging.info(f"{BOLD}Version:{RESET} {LIGHT_YELLOW}sptnr v{__version__}{RESET}")
+
 if args.preview:
     logging.info(f"{LIGHT_YELLOW}Preview mode, no changes will be made.{RESET}")
     PREVIEW = 1
@@ -159,6 +185,20 @@ if not args.preview:
     )
 
 
+def validate_url(url):
+    if not re.match(r"https?://", url):
+        logging.error(
+            f"{LIGHT_RED}Config Error: URL must start with 'http://' or 'https://'.{RESET}"
+        )
+        return False
+    if url.endswith("/"):
+        logging.error(
+            f"{LIGHT_RED}Config Error: URL must not end with a trailing slash.{RESET}"
+        )
+        return False
+    return True
+
+
 def url_encode(string):
     return urllib.parse.quote_plus(string)
 
@@ -187,24 +227,24 @@ def process_track(track_id, artist_name, album, track_name):
         try:
             response = requests.get(spotify_url, headers=headers)
         except requests.exceptions.ConnectionError:
-            logging.error(f"{LIGHT_RED}Error: Unable to reach server.{RESET}")
+            logging.error(f"{LIGHT_RED}Spotify Error: Unable to reach server.{RESET}")
             sys.exit(1)
 
         if response.status_code != 200:
             if response.status_code == 429:
                 logging.error(
-                    f"{LIGHT_RED}Error {response.status_code}: Retry after {BOLD}{response.headers.get('Retry-After', 'some time')}s{RESET}"
+                    f"{LIGHT_RED}Spotify Error {response.status_code}: Retry after {BOLD}{response.headers.get('Retry-After', 'some time')}s{RESET}"
                 )
             else:
                 logging.error(
-                    f"{LIGHT_RED}Error {response.status_code}: {response.text}{RESET}"
+                    f"{LIGHT_RED}Spotify Error {response.status_code}: {response.text}{RESET}"
                 )
             sys.exit(1)
         try:
             return response.json()
         except ValueError as e:
             logging.error(
-                f"{LIGHT_RED}Error decoding JSON from Spotify API: {e}{RESET}"
+                f"{LIGHT_RED}Spotify Error: Error decoding JSON from Spotify API: {e}{RESET}"
             )
             sys.exit(1)
 
@@ -231,6 +271,15 @@ def process_track(track_id, artist_name, album, track_name):
         )
         found_track = len(spotify_data.get("tracks", {}).get("items", [])) > 0
 
+    if not found_track:
+        # Tertiary Search (replace 'Part' with 'Pt.')
+        modified_track_name = track_name.replace("Part", "Pt.")
+        encoded_modified_track_name = url_encode(modified_track_name)
+        spotify_data = search_spotify(
+            f"{encoded_modified_track_name}%20artist:{encoded_artist_name}"
+        )
+        found_track = len(spotify_data.get("tracks", {}).get("items", []))
+
     if found_track:
         popularity = spotify_data["tracks"]["items"][0].get("popularity", 0)
         rating = get_rating_from_popularity(popularity)
@@ -256,6 +305,13 @@ def process_track(track_id, artist_name, album, track_name):
 
 
 def process_album(album_id):
+    if not args.force:
+        global processed_albums
+
+        if album_id in processed_albums:
+            logging.info(f"    {LIGHT_CYAN}Skipping already processed album{RESET}")
+            return
+
     nav_url = f"{NAV_BASE_URL}/rest/getAlbum?id={album_id}&u={NAV_USER}&p=enc:{HEX_ENCODED_PASS}&v=1.12.0&c=spotify_sync&f=json"
     response = requests.get(nav_url).json()
 
@@ -269,6 +325,10 @@ def process_album(album_id):
     for track in tracks:
         process_track(*track)
 
+    processed_albums.add(album_id)
+    with open(PROCESSED_ALBUMS_FILE, "a") as file:
+        file.write(f"{album_id}\n")
+
 
 def process_artist(artist_id):
     nav_url = f"{NAV_BASE_URL}/rest/getArtist?id={artist_id}&u={NAV_USER}&p=enc:{HEX_ENCODED_PASS}&v=1.12.0&c=spotify_sync&f=json"
@@ -285,9 +345,52 @@ def process_artist(artist_id):
 
 
 def fetch_data(url):
-    response = requests.get(url)
-    return json.loads(response.text)["subsonic-response"]
+    try:
+        response = requests.get(url)
+        response_data = json.loads(response.text)
 
+        if "subsonic-response" not in response_data:
+            logging.error(
+                f"{LIGHT_RED}Unexpected response format from Navidrome.{RESET}"
+            )
+            sys.exit(1)
+
+        nav_response = response_data["subsonic-response"]
+
+        if "error" in nav_response:
+            error_message = nav_response["error"].get("message", "Unknown error")
+            logging.error(f"{LIGHT_RED}Navidrome Error: {error_message}{RESET}")
+            sys.exit(1)
+
+        return nav_response
+
+    except requests.exceptions.ConnectionError:
+        logging.error(
+            f"{LIGHT_RED}Connection Error: Failed to connect to the provided URL. Please check if the URL is correct and the server is reachable.{RESET}"
+        )
+        sys.exit(1)
+    except requests.exceptions.RequestException as e:
+        logging.error(
+            f"{LIGHT_RED}Connection Error: An error occurred while trying to connect to Navidrome: {e}{RESET}"
+        )
+        sys.exit(1)
+    except json.JSONDecodeError:
+        logging.error(
+            f"{LIGHT_RED}JSON Parsing Error: Failed to parse JSON response from Navidrome. Please check if the provided URL is a valid Navidrome server.{RESET}"
+        )
+        sys.exit(1)
+
+
+# Load processed albums
+if os.path.exists(PROCESSED_ALBUMS_FILE) and not args.force:
+    with open(PROCESSED_ALBUMS_FILE, "r") as file:
+        processed_albums = set(file.read().splitlines())
+
+try:
+    validate_url(NAV_BASE_URL)
+except ValueError as e:
+    logging.error(f"{LIGHT_RED}{e}{RESET}")
+    sys.exit(1)
 
 if ARTIST_IDs:
     for ARTIST_ID in ARTIST_IDs:
@@ -334,7 +437,7 @@ else:
     logging.info(f"Total artists to process: {LIGHT_GREEN}{total_count}{RESET}")
 
     for index, ARTIST_ENTRY in tqdm(
-        enumerate(data_slice), total=total_count, leave=False
+        enumerate(data_slice), total=total_count, leave=False, unit="artist"
     ):
         ARTIST_ID, ARTIST_NAME = ARTIST_ENTRY
 
@@ -349,14 +452,26 @@ else:
 
 # Display the results
 logging.info("")
-MATCH_PERCENTAGE = (FOUND_AND_UPDATED / TOTAL_TRACKS) * 100 if TOTAL_TRACKS != 0 else 0
-FORMATTED_MATCH_PERCENTAGE = round(MATCH_PERCENTAGE, 2)  # Rounding to 2 decimal places
+
+# Check if TOTAL_TRACKS is zero to avoid division by zero error
+if TOTAL_TRACKS > 0:
+    MATCH_PERCENTAGE = (FOUND_AND_UPDATED / TOTAL_TRACKS) * 100
+else:
+    MATCH_PERCENTAGE = 0
+
+FORMATTED_MATCH_PERCENTAGE = round(MATCH_PERCENTAGE, 2)
 TOTAL_BLOCKS = 20
 
 color_found = LIGHT_GREEN if FOUND_AND_UPDATED == TOTAL_TRACKS else LIGHT_YELLOW
 color_found_white = LIGHT_GREEN if FOUND_AND_UPDATED == TOTAL_TRACKS else BOLD
 color_not_found = LIGHT_GREEN if NOT_FOUND == 0 else LIGHT_RED
-blocks_found = "█" * round(FOUND_AND_UPDATED * TOTAL_BLOCKS / TOTAL_TRACKS)
+
+# Adjust the progress bar calculation
+blocks_found = (
+    "█" * round(FOUND_AND_UPDATED * TOTAL_BLOCKS / TOTAL_TRACKS)
+    if TOTAL_TRACKS > 0
+    else ""
+)
 blocks_not_found = "█" * (TOTAL_BLOCKS - len(blocks_found))
 full_blocks_found = f"{color_found_white}{blocks_found}{RESET}"
 full_blocks_not_found = f"{color_not_found}{blocks_not_found}{RESET}"
@@ -376,7 +491,6 @@ if seconds or not parts:  # Show seconds if it's the only value, even if it's 0
 
 formatted_elapsed_time = " ".join(parts)
 
-# logging.info(f"Processing completed in {int(hours):02}:{int(minutes):02}:{int(seconds):02}")
 logging.info(
     f"Tracks: {LIGHT_PURPLE}{TOTAL_TRACKS}{RESET} | Found: {color_found}{FOUND_AND_UPDATED}{RESET} |{full_blocks_found}{full_blocks_not_found}| Not Found: {color_not_found}{NOT_FOUND}{RESET} | Match: {color_found}{FORMATTED_MATCH_PERCENTAGE}%{RESET} | Time: {LIGHT_PURPLE}{formatted_elapsed_time}{RESET}"
 )