feat: better logs view

.env.example

@@ -3,3 +3,5 @@ NAV_USER=your_navidrome_username
 NAV_PASS=your_navidrome_password
 SPOTIFY_CLIENT_ID=your_spotify_client_id
 SPOTIFY_CLIENT_SECRET=your_spotify_client_secret
+WEB_API_KEY=changeme
+ENABLE_WEB_API_KEY=True

.gitignore

@@ -1,3 +1,4 @@
-logs/*.log
+data/
 .env
 docker-compose.yml
+__pycache__

Dockerfile

@@ -10,5 +10,8 @@ COPY . .
 # Install any needed packages specified in requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Use an entrypoint script
+# Expose port 5000 for the Flask app
-ENTRYPOINT ["python", "./sptnr.py"]
+EXPOSE 5000
+
+# Set the entrypoint to Python
+ENTRYPOINT ["python", "sptnr.py"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Kevin Restaino
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

VERSION

@@ -1 +1 @@
-1.1.1
+2.0.0-alpha

build_and_push.sh

@@ -6,14 +6,14 @@ set -e
 # Read version from the VERSION file
 VERSION=$(cat VERSION)
 
-# Build the Docker image with the version tag
+# Set up the builder instance (only needs to be done once, so you can comment this out after the first run)
-docker build -t krestaino/sptnr:$VERSION .
+# docker buildx create --name mybuilder --use
+# docker buildx inspect mybuilder --bootstrap
 
-# Tag the built image as latest
+# Build and push the Docker image for both arm64 and amd64 platforms with the version tag
-docker tag krestaino/sptnr:$VERSION krestaino/sptnr:latest
+docker buildx build --platform linux/arm64,linux/amd64 -t krestaino/sptnr:$VERSION . --push
 
-# Push both tags to the Docker registry
+# Build and push the 'latest' tag as well
-docker push krestaino/sptnr:$VERSION
+docker buildx build --platform linux/arm64,linux/amd64 -t krestaino/sptnr:latest . --push
-docker push krestaino/sptnr:latest
 
 echo "Docker images tagged and pushed: $VERSION and latest"

docker-compose.yml.example

@@ -9,11 +9,21 @@ services:
     # Uncomment the next line to build the Docker image locally
     # build: .
 
+    # Uncomment the next line to run the script
+    # entrypoint: ["python", "sptnr.py"]
+    
+    # Uncomment the next line to start the web server
+    # entrypoint: ["gunicorn", "-b", "0.0.0.0:5000", "server:sptnr"]
+
     environment:
       - NAV_BASE_URL=your_navidrome_server_url
       - NAV_USER=your_navidrome_username
       - NAV_PASS=your_navidrome_password
       - SPOTIFY_CLIENT_ID=your_spotify_client_id
       - SPOTIFY_CLIENT_SECRET=your_spotify_client_secret
+      - WEB_API_KEY=changeme
+      - ENABLE_WEB_API_KEY=False
     volumes:
-      - ./logs:/usr/src/app/logs
+      - ./data:/usr/src/app/data
+    ports:
+      - "3333:5000"

requirements.txt

@@ -2,3 +2,6 @@ requests==2.31.0
 python-dotenv==1.0.0
 colorama==0.4.6
 tqdm==4.66.1
+gunicorn==20.1.0
+Flask==2.1.2
+Werkzeug==2.0.3

server.py

@@ -0,0 +1,133 @@
+from flask import Flask, request, jsonify
+import subprocess
+import threading
+import os
+from dotenv import load_dotenv
+import functools
+import re
+import datetime
+
+load_dotenv()
+
+sptnr = Flask(__name__)
+WEB_API_KEY = os.getenv("WEB_API_KEY")
+ENABLE_WEB_API_KEY = os.getenv("ENABLE_WEB_API_KEY", "True") == "True"
+LOG_DIR = "data/logs"
+
+
+def api_key_required(f):
+    @functools.wraps(f)
+    def decorated_function(*args, **kwargs):
+        if ENABLE_WEB_API_KEY and request.args.get("api_key") != WEB_API_KEY:
+            return jsonify({"error": "Unauthorized"}), 401
+        return f(*args, **kwargs)
+
+    return decorated_function
+
+
+def run_script(cmd):
+    subprocess.run(cmd)
+
+
+@sptnr.route("/process", methods=["GET", "POST"])
+@api_key_required
+def process_request():
+    cmd = ["python3", "sptnr.py"]
+    if request.args.get("preview", "") == "true":
+        cmd.append("--preview")
+    if request.args.get("force", "") == "true":
+        cmd.append("--force")
+    artist_ids = request.args.getlist("artist")
+    for artist_id in artist_ids:
+        cmd.extend(["--artist", artist_id])
+    album_ids = request.args.getlist("album")
+    for album_id in album_ids:
+        cmd.extend(["--album", album_id])
+    start = request.args.get("start")
+    if start:
+        cmd.extend(["--start", start])
+    limit = request.args.get("limit")
+    if limit:
+        cmd.extend(["--limit", limit])
+
+    thread = threading.Thread(target=run_script, args=(cmd,))
+    thread.start()
+
+    return jsonify({"message": "Processing started"})
+
+
+@sptnr.route("/logs")
+@api_key_required
+def list_logs():
+    try:
+        logs = os.listdir(LOG_DIR)
+        logs = [log for log in logs if log.endswith(".log")]  # Filter log files
+        full_paths = [os.path.join(LOG_DIR, log) for log in logs]
+        logs_sorted = sorted(full_paths, key=os.path.getmtime, reverse=True)
+
+        table_rows = []
+        for log_path in logs_sorted:
+            with open(log_path, "r") as file:
+                last_line = file.readlines()[-1]
+                tracks, found, not_found, match, time = parse_log_data(last_line)
+
+            log_name = os.path.basename(log_path)
+            timestamp = int(log_name.split("_")[1].split(".")[0])
+            log_datetime = datetime.datetime.fromtimestamp(timestamp)
+            formatted_datetime = log_datetime.strftime("%Y-%m-%d %H:%M:%S")
+
+            table_rows.append(
+                f"<tr><td><a href='/logs/{log_name}'>{log_name}</a></td><td>{formatted_datetime}</td><td>{tracks}</td><td>{found}</td><td>{not_found}</td><td>{match}</td><td>{time}</td></tr>"
+            )
+
+        table_html = f"<table border='1'><tr><th>Log File</th><th>Date & Time</th><th>Tracks</th><th>Found</th><th>Not Found</th><th>Match</th><th>Time</th></tr>{''.join(table_rows)}</table>"
+        return table_html
+    except Exception as e:
+        return f"An error occurred: {e}", 500
+
+
+def parse_log_data(line):
+    # Regular expressions to extract the required data
+    tracks_pattern = r"Tracks: (\d+)"
+    found_pattern = r"Found: (\d+)"
+    not_found_pattern = r"Not Found: (\d+)"
+    match_pattern = r"Match: ([\d.]+%)"
+    time_pattern = r"Time: ([\ds]+)"
+
+    # Extracting data using regular expressions
+    tracks = re.search(tracks_pattern, line)
+    found = re.search(found_pattern, line)
+    not_found = re.search(not_found_pattern, line)
+    match = re.search(match_pattern, line)
+    time = re.search(time_pattern, line)
+
+    # Getting the values or "N/A" if not found
+    tracks = tracks.group(1) if tracks else "N/A"
+    found = found.group(1) if found else "N/A"
+    not_found = not_found.group(1) if not_found else "N/A"
+    match = match.group(1) if match else "N/A"
+    time = time.group(1) if time else "N/A"
+
+    return tracks, found, not_found, match, time
+
+
+@sptnr.route("/logs/<filename>")
+@api_key_required
+def view_log(filename):
+    try:
+        full_path = os.path.join(LOG_DIR, filename)
+        if not os.path.exists(full_path) or not os.path.isfile(full_path):
+            return "File not found", 404
+
+        with open(full_path, "r") as file:
+            content = file.read()
+
+        # Convert content to HTML-friendly format
+        content = content.replace("\n", "<br>")
+        return f"<pre>{content}</pre>"
+    except Exception as e:
+        return f"An error occurred: {e}", 500
+
+
+if __name__ == "__main__":
+    sptnr.run(debug=False, host="0.0.0.0", port=3333)

sptnr.py

@@ -41,11 +41,11 @@ BOLD = Style.BRIGHT
 RESET = Style.RESET_ALL
 
 # Setup logs
-LOG_DIR = "logs"
+LOG_DIR = "data/logs"
 if not os.path.exists(LOG_DIR):
     os.makedirs(LOG_DIR)
 
-LOGFILE = os.path.join(LOG_DIR, f"spotify-popularity_{int(time.time())}.log")
+LOGFILE = os.path.join(LOG_DIR, f"sptnr_{int(time.time())}.log")
 
 
 class NoColorFormatter(logging.Formatter):
@@ -103,6 +103,9 @@ TOTAL_TRACKS = 0
 FOUND_AND_UPDATED = 0
 NOT_FOUND = 0
 UNMATCHED_TRACKS = []
+PROCESSED_ALBUMS_FILE = "data/processed_albums.txt"
+
+processed_albums = set()
 
 # Parse arguments
 description_text = "process command-line flags for sync"
@@ -147,6 +150,13 @@ parser.add_argument(
     "-v", "--version", action="version", version=f"%(prog)s {__version__}"
 )
 
+parser.add_argument(
+    "-f",
+    "--force",
+    action="store_true",
+    help="force processing of all albums, even if they were processed previously",
+)
+
 
 args = parser.parse_args()
 
@@ -261,6 +271,15 @@ def process_track(track_id, artist_name, album, track_name):
         )
         found_track = len(spotify_data.get("tracks", {}).get("items", [])) > 0
 
+    if not found_track:
+        # Tertiary Search (replace 'Part' with 'Pt.')
+        modified_track_name = track_name.replace("Part", "Pt.")
+        encoded_modified_track_name = url_encode(modified_track_name)
+        spotify_data = search_spotify(
+            f"{encoded_modified_track_name}%20artist:{encoded_artist_name}"
+        )
+        found_track = len(spotify_data.get("tracks", {}).get("items", []))
+
     if found_track:
         popularity = spotify_data["tracks"]["items"][0].get("popularity", 0)
         rating = get_rating_from_popularity(popularity)
@@ -286,6 +305,13 @@ def process_track(track_id, artist_name, album, track_name):
 
 
 def process_album(album_id):
+    if not args.force:
+        global processed_albums
+
+        if album_id in processed_albums:
+            logging.info(f"    {LIGHT_CYAN}Skipping already processed album{RESET}")
+            return
+
     nav_url = f"{NAV_BASE_URL}/rest/getAlbum?id={album_id}&u={NAV_USER}&p=enc:{HEX_ENCODED_PASS}&v=1.12.0&c=spotify_sync&f=json"
     response = requests.get(nav_url).json()
 
@@ -299,6 +325,10 @@ def process_album(album_id):
     for track in tracks:
         process_track(*track)
 
+    processed_albums.add(album_id)
+    with open(PROCESSED_ALBUMS_FILE, "a") as file:
+        file.write(f"{album_id}\n")
+
 
 def process_artist(artist_id):
     nav_url = f"{NAV_BASE_URL}/rest/getArtist?id={artist_id}&u={NAV_USER}&p=enc:{HEX_ENCODED_PASS}&v=1.12.0&c=spotify_sync&f=json"
@@ -351,6 +381,11 @@ def fetch_data(url):
         sys.exit(1)
 
 
+# Load processed albums
+if os.path.exists(PROCESSED_ALBUMS_FILE) and not args.force:
+    with open(PROCESSED_ALBUMS_FILE, "r") as file:
+        processed_albums = set(file.read().splitlines())
+
 try:
     validate_url(NAV_BASE_URL)
 except ValueError as e:
@@ -402,7 +437,7 @@ else:
     logging.info(f"Total artists to process: {LIGHT_GREEN}{total_count}{RESET}")
 
     for index, ARTIST_ENTRY in tqdm(
-        enumerate(data_slice), total=total_count, leave=False
+        enumerate(data_slice), total=total_count, leave=False, unit="artist"
     ):
         ARTIST_ID, ARTIST_NAME = ARTIST_ENTRY
 
@@ -417,14 +452,26 @@ else:
 
 # Display the results
 logging.info("")
-MATCH_PERCENTAGE = (FOUND_AND_UPDATED / TOTAL_TRACKS) * 100 if TOTAL_TRACKS != 0 else 0
+
-FORMATTED_MATCH_PERCENTAGE = round(MATCH_PERCENTAGE, 2)  # Rounding to 2 decimal places
+# Check if TOTAL_TRACKS is zero to avoid division by zero error
+if TOTAL_TRACKS > 0:
+    MATCH_PERCENTAGE = (FOUND_AND_UPDATED / TOTAL_TRACKS) * 100
+else:
+    MATCH_PERCENTAGE = 0
+
+FORMATTED_MATCH_PERCENTAGE = round(MATCH_PERCENTAGE, 2)
 TOTAL_BLOCKS = 20
 
 color_found = LIGHT_GREEN if FOUND_AND_UPDATED == TOTAL_TRACKS else LIGHT_YELLOW
 color_found_white = LIGHT_GREEN if FOUND_AND_UPDATED == TOTAL_TRACKS else BOLD
 color_not_found = LIGHT_GREEN if NOT_FOUND == 0 else LIGHT_RED
-blocks_found = "█" * round(FOUND_AND_UPDATED * TOTAL_BLOCKS / TOTAL_TRACKS)
+
+# Adjust the progress bar calculation
+blocks_found = (
+    "█" * round(FOUND_AND_UPDATED * TOTAL_BLOCKS / TOTAL_TRACKS)
+    if TOTAL_TRACKS > 0
+    else ""
+)
 blocks_not_found = "█" * (TOTAL_BLOCKS - len(blocks_found))
 full_blocks_found = f"{color_found_white}{blocks_found}{RESET}"
 full_blocks_not_found = f"{color_not_found}{blocks_not_found}{RESET}"
@@ -444,7 +491,6 @@ if seconds or not parts:  # Show seconds if it's the only value, even if it's 0
 
 formatted_elapsed_time = " ".join(parts)
 
-# logging.info(f"Processing completed in {int(hours):02}:{int(minutes):02}:{int(seconds):02}")
 logging.info(
     f"Tracks: {LIGHT_PURPLE}{TOTAL_TRACKS}{RESET} | Found: {color_found}{FOUND_AND_UPDATED}{RESET} |{full_blocks_found}{full_blocks_not_found}| Not Found: {color_not_found}{NOT_FOUND}{RESET} | Match: {color_found}{FORMATTED_MATCH_PERCENTAGE}%{RESET} | Time: {LIGHT_PURPLE}{formatted_elapsed_time}{RESET}"
 )