Initial commit

1 files changed, 567 insertions, 0 deletions

diff --git a/lib/newgrounds/newgrounds.py b/lib/newgrounds/newgrounds.py
new file mode 100644
index 0000000..10a5b57
--- /dev/null
+++ b/lib/newgrounds/newgrounds.py
@@ -0,0 +1,567 @@
+# Copyright © 2024 Jakob L. Kreuze <zerodaysfordays@sdf.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""
+Video plugin for animations posted to Newgrounds.
+"""
+
+import concurrent.futures
+from datetime import datetime
+from dataclasses import dataclass, field
+from enum import Enum
+
+from dateutil import parser as dateutil_parser
+import json
+import os
+import sys
+from typing import List
+from urllib.parse import urlencode, parse_qsl
+
+from bs4 import BeautifulSoup
+import requests
+
+Suitability = Enum("Suitability", ["E", "T", "M", "A"])
+Conduct = Enum(
+    "Conduct",
+    ["MOVIES", "AUDIO", "ART", "ALBUMS", "PLAYLISTS", "COLLECTIONS", "SERIES"],
+)
+Grouping = Enum("Grouping", ["FEATURED", "LATEST", "POPULAR"])
+SortBy = Enum(
+    "SortBy",
+    [
+        "RELEVANCE",
+        "DATE_ASC",
+        "DATE_DESC",
+        "SCORE_ASC",
+        "SCORE_DESC",
+        "VIEWS_ASC",
+        "VIEWS_DESC",
+    ],
+)
+Interval = Enum("Interval", ["TODAY", "YESTERDAY", "WEEK", "MONTH", "YEAR"])
+AudioArchetype = Enum("AudioArchetype", ["MUSIC", "VOICE", "PODCASTS"])
+
+
+def stringify_enum(enum_value):
+    return enum_value.name.lower().replace("_", "-")
+
+
+@dataclass
+class VisualLink:
+    title: str
+    thumbnail: str
+    author: str
+    url: str
+
+
+@dataclass
+class Movie:
+    title: str
+    thumbnail: str
+    content_url: str
+    url: str
+    views: int = 0
+    faves: int = 0
+    votes: int = 0
+    score: float = 0.0
+    upload_date: datetime = datetime.fromtimestamp(0)
+    genre: str = "Unknown"
+    tags: List[str] = field(default_factory=lambda: [])
+    description: str = ""
+    authors: List[str] = field(default_factory=lambda: [])
+    suitability: Suitability = Suitability.E
+
+    def __post_init__(self):
+        pass
+
+    def url(self):
+        pass
+
+
+@dataclass
+class Audio:
+    title: str
+    thumbnail: str
+    content_url: str
+    url: str
+    views: int = 0
+    faves: int = 0
+    votes: int = 0
+    score: float = 0.0
+    upload_date: datetime = datetime.fromtimestamp(0)
+    genre: str = "Unknown"
+    tags: list[str] = field(default_factory=lambda: [])
+    description: str = ""
+    authors: list[str] = field(default_factory=lambda: [])
+    suitability: Suitability = Suitability.E
+
+    def __post_init__(self):
+        pass
+
+    def url(self):
+        pass
+
+
+MOVIE_CATEGORIES = {
+    "All": 0,
+    "Action": 45,
+    "Comedy - Original": 60,
+    "Comedy - Parody": 61,
+    "Drama": 47,
+    "Experimental": 49,
+    "Informative": 48,
+    "Music Video": 50,
+    "Other": 51,
+    "Spam": 55,
+}
+
+AUDIO_CATEGORIES = {
+    "All": 0,
+    "Easy Listening - Classical": 3,
+    "Easy Listening - Jazz": 18,
+    "Easy Listening - Solo Instrument": 51,
+    "Electronic - Ambient": 5,
+    "Electronic - Chipstep": 48,
+    "Electronic - Dance": 6,
+    "Electronic - Drum N Bass": 7,
+    "Electronic - Dubstep": 41,
+    "Electronic - House": 9,
+    "Electronic - Industrial": 8,
+    "Electronic - New Wave": 20,
+    "Electronic - Synthwave": 57,
+    "Electronic - Techno": 10,
+    "Electronic - Trance": 11,
+    "Electronic - Video Game": 12,
+    "Hip Hop, Rap, R&amp;B - Hip Hop - Modern": 17,
+    "Hip Hop, Rap, R&amp;B - Hip Hop - Olskool": 16,
+    "Hip Hop, Rap, R&amp;B - Nerdcore": 47,
+    "Hip Hop, Rap, R&amp;B - R&amp;B": 21,
+    "Metal, Rock - Brit Pop": 22,
+    "Metal, Rock - Classic Rock": 23,
+    "Metal, Rock - General Rock": 24,
+    "Metal, Rock - Grunge": 25,
+    "Metal, Rock - Heavy Metal": 15,
+    "Metal, Rock - Indie": 26,
+    "Metal, Rock - Pop": 27,
+    "Metal, Rock - Punk": 28,
+    "Other - Cinematic": 50,
+    "Other - Experimental": 49,
+    "Other - Funk": 13,
+    "Other - Fusion": 52,
+    "Other - Goth": 14,
+    "Other - Miscellaneous": 39,
+    "Other - Ska": 29,
+    "Other - World": 19,
+    "Southern Flavor - Bluegrass": 1,
+    "Southern Flavor - Blues": 2,
+    "Southern Flavor - Country": 4,
+}
+
+ART_CATEGORIES = {
+    "All": 0,
+}
+
+SESSION_COOKIE_NAME = "vmkIdu5l8m"
+
+
+class Newgrounds(object):
+    def __init__(self):
+        pass
+
+    def _request(self, method, url, **kwargs):
+        kwargs = dict(kwargs)
+        # TODO: Test if we need to open a new session?
+        if hasattr(self, "session") and self.session is not None:
+            if "cookies" in kwargs:
+                kwargs["cookies"][SESSION_COOKIE_NAME] = self.session
+            else:
+                kwargs["cookies"] = {SESSION_COOKIE_NAME: self.session}
+        r = requests.request(method, url, **kwargs)
+        return r
+
+    def login(self, username, password):
+        self.session = None
+        html = self._request("GET", "https://www.newgrounds.com/login").text
+        soup = BeautifulSoup(html, "html.parser")
+
+        form = soup.find("form", method="post")
+        endpoint = form["action"]
+        auth = form.find("input")["value"]
+
+        r = self._request(
+            "POST",
+            "https://www.newgrounds.com" + endpoint,
+            data={
+                "auth": auth,
+                "username": username,
+                "password": password,
+                "code": "",
+                "codehint": "------",
+            },
+            cookies={
+                "passport-auth": auth,
+            },
+        )
+        self.session = r.cookies[SESSION_COOKIE_NAME]
+
+    def _test_logged_in(session):
+        return (
+            requests.get(
+                "https://www.newgrounds.com/account",
+                cookies={SESSION_COOKIE_NAME: session},
+            ).status_code
+            != 401
+        )
+
+    def series(self, official=True, sort_by=SortBy.DATE_DESC, offset=1):
+        html = self._request(
+            "GET",
+            "https://www.newgrounds.com/series",
+            params={
+                "filter": "all" if not official else None,
+                "order": stringify_enum(sort_by),
+                "page": offset,
+            },
+        ).text
+        soup = BeautifulSoup(html, "html.parser")
+        visual_links = [
+            json.loads(li["data-visual-link"])
+            for li in soup.find_all("li", "visual-link-container")
+        ]
+        return self._fetch_visual_links(visual_links)
+
+    def collections(self, official=True, sort_by=SortBy.DATE_DESC, offset=1):
+        html = self._request(
+            "GET",
+            "https://www.newgrounds.com/collections",
+            params={
+                "filter": "all" if not official else None,
+                "order": stringify_enum(sort_by),
+                "page": offset,
+            },
+        ).text
+        soup = BeautifulSoup(html, "html.parser")
+        visual_links = [
+            json.loads(li["data-visual-link"])
+            for li in soup.find_all("li", "visual-link-container")
+        ]
+        return self._fetch_visual_links(visual_links)
+
+    def playlist_entries(self, url):
+        html = self._request("GET", url).text
+        soup = BeautifulSoup(html, "html.parser")
+        visual_links = [
+            json.loads(li["data-visual-link"])
+            for li in soup.find_all("li", "visual-link-container")
+        ]
+        urls = [card["url"] for card in self._fetch_visual_links(visual_links)]
+        return self._fetch_metadata_batch(urls)
+
+    def _fetch_metadata_batch(self, urls):
+        result = []
+        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+            # Start the load operations and mark each future with its URL
+            future_to_url = {
+                executor.submit(self._fetch_metadata, url): url for url in urls
+            }
+            for future in concurrent.futures.as_completed(future_to_url):
+                url = future_to_url[future]
+                result.append(future.result())
+        return result
+
+    def _fetch_visual_links(self, visual_links):
+        r = self._request(
+            "POST",
+            "https://www.newgrounds.com/visual-links-fetch",
+            params={
+                "X-Requested-With": "XMLHttpRequest",
+            },
+            data={
+                "ids": json.dumps(visual_links).replace(" ", ""),
+                "component_params[include_author]": "1",
+                "include_all_suitabilities": "0",
+                "isAjaxRequest": "1",
+            },
+        )
+        result = r.json()
+        entries = []
+        if isinstance(result["partials"], dict):
+            for partial in result["partials"].values():
+                if isinstance(partial, dict):
+                    for partial in partial.values():
+                        soup = BeautifulSoup(partial, "html.parser")
+                        title = soup.find("h4").string
+                        thumbnail = soup.find("img")["src"]
+                        author = soup.find("strong").string
+                        url = soup.find("a")["href"]
+                        entries.append(VisualLink(title, thumbnail, author, url))
+        return entries
+
+    def _fetch_movie_content_url(self, movie_id):
+        response = self._request(
+            "GET",
+            "https://www.newgrounds.com/portal/video/{}".format(movie_id),
+            headers={"X-Requested-With": "XMLHttpRequest"},
+        ).json()
+        preferences = ["1080p", "720p", "360p"]
+        for preference in preferences:
+            if preference in response["sources"]:
+                source = next(
+                    filter(
+                        lambda x: x["type"] == "video/mp4",
+                        response["sources"][preference],
+                    )
+                )
+                if source is not None:
+                    return source["src"]
+
+    def _fetch_movie_metadata(self, url):
+        html = self._request("GET", url).text
+        soup = BeautifulSoup(html, "html.parser")
+
+        title = soup.find("div", id="embed_header").h2.string
+        thumbnail = soup.find("meta", property="og:image")
+        if thumbnail is not None:
+            thumbnail = thumbnail["content"]
+
+        movie_id = soup.find("meta", property="og:url")["content"].strip(
+            "https://www.newgrounds.com/portal/view/"
+        )
+        content_url = self._fetch_movie_content_url(movie_id)
+
+        metadata = {
+            "title": title,
+            "thumbnail": thumbnail,
+            "content_url": content_url,
+            "url": url,
+        }
+
+        sidestats_popularity = soup.find_all("dl", "sidestats")[0].find_all("dd")
+        if len(sidestats_popularity) == 4:
+            metadata["views"] = int(sidestats_popularity[0].string.replace(",", ""))
+            faves = sidestats_popularity[1].a
+            if faves is not None:
+                metadata["faves"] = int(faves.string.replace(",", ""))
+            metadata["votes"] = int(sidestats_popularity[2].string.replace(",", ""))
+            score = sidestats_popularity[3].span
+            if score is not None:
+                metadata["score"] = float(score.string)
+
+        sidestats_date_and_genre = soup.find_all("dl", "sidestats")[1].find_all("dd")
+        if len(sidestats_date_and_genre) == 3:
+            metadata["upload_date"] = dateutil_parser.parse(
+                sidestats_date_and_genre[0].string
+                + " "
+                + sidestats_date_and_genre[1].string
+            )
+            genre = sidestats_date_and_genre[2].a
+            if genre is not None:
+                metadata["genre"] = genre.string
+
+        tags = soup.find("dd", "tags")
+        if tags is not None:
+            metadata["tags"] = [tag.string for tag in tags.find_all("a")]
+
+        author_comments = soup.find("div", id="author_comments")
+        metadata["description"] = " ".join([s for s in author_comments.strings]).strip()
+
+        authors = soup.find("ul", "authorlinks")
+        metadata["authors"] = [
+            author.string
+            for author in authors.find_all("a")
+            if author.get("class") is None
+        ]
+
+        if len(soup.find_all("h2", "rated-a")) != 0:
+            rating = "a"
+        elif len(soup.find_all("h2", "rated-m")) != 0:
+            rating = "m"
+        elif len(soup.find_all("h2", "rated-t")) != 0:
+            rating = "t"
+        else:
+            rating = "e"
+
+        return Movie(**metadata)
+
+    def _fetch_audio_metadata(self, url):
+        html = self._request("GET", url).text
+        soup = BeautifulSoup(html, "html.parser")
+
+        title = soup.find("div", "pod-head").h2.string
+        thumbnail = soup.find("meta", property="og:image")
+        if thumbnail is not None:
+            thumbnail = thumbnail["content"]
+        content_url = soup.find("meta", property="og:audio")["content"]
+
+        metadata = {
+            "title": title,
+            "thumbnail": thumbnail,
+            "content_url": content_url,
+            "url": url,
+        }
+
+        sidestats_popularity = soup.find_all("dl", "sidestats")[0].find_all("dd")
+        if len(sidestats_popularity) == 4:
+            metadata["views"] = int(sidestats_popularity[0].string.replace(",", ""))
+            faves = sidestats_popularity[1].a
+            if faves is not None:
+                metadata["faves"] = int(faves.string.replace(",", ""))
+            metadata["votes"] = int(sidestats_popularity[2].string.replace(",", ""))
+            score = sidestats_popularity[3].span
+            if score is not None:
+                metadata["score"] = float(score.string)
+
+        sidestats_date_and_genre = soup.find_all("dl", "sidestats")[1].find_all("dd")
+        if len(sidestats_date_and_genre) >= 3:
+            metadata["upload_date"] = dateutil_parser.parse(
+                sidestats_date_and_genre[0].string
+                + " "
+                + sidestats_date_and_genre[1].string
+            )
+            genre = sidestats_date_and_genre[2].a
+            if genre is not None:
+                metadata["genre"] = genre.string
+
+        if len(sidestats_date_and_genre) >= 5:
+            metadata["length"] = sidestats_date_and_genre[5].string.strip()
+
+        tags = soup.find("dd", "tags")
+        if tags is not None:
+            metadata["tags"] = [tag.string for tag in tags.find_all("a")]
+
+        # author_comments = soup.find("div", id="author_comments")
+        # metadata["description"] = " ".join([s for s in author_comments.strings]).strip()
+
+        authors = soup.find("div", "item-user")
+        metadata["author"] = [
+            author.string
+            for author in authors.find_all("a")
+            if author.get("class") is None
+        ][0]
+
+        if len(soup.find_all("h2", "rated-a")) != 0:
+            rating = "a"
+        elif len(soup.find_all("h2", "rated-m")) != 0:
+            rating = "m"
+        elif len(soup.find_all("h2", "rated-t")) != 0:
+            rating = "t"
+        else:
+            rating = "e"
+
+        return Audio(**metadata)
+
+    def _fetch_metadata(self, url):
+        if "newgrounds.com/audio/" in url:
+            return self._fetch_audio_metadata(url)
+        elif "newgrounds.com/art/" in url:
+            return self._fetch_art_metadata(url)
+        else:
+            return self._fetch_movie_metadata(url)
+
+    def search(self, term, conduct=Conduct.MOVIES, sort_by=SortBy.RELEVANCE, offset=1):
+        """
+        Return a list of URLs matching a given search term.
+        """
+        response = self._request(
+            "GET",
+            "https://www.newgrounds.com/search/conduct/{}".format(
+                stringify_enum(conduct)
+            ),
+            params={
+                "suitabilities": "etm",
+                "sort": stringify_enum(sort_by),
+                "terms": term,
+                "page": offset,
+                "inner": 1,
+            },
+            headers={
+                "X-Requested-With": "XMLHttpRequest",
+            },
+        ).json()
+
+        soup = BeautifulSoup(response["content"], "html.parser")
+        urls = [tag["href"] for tag in soup.find_all("a", "item-portalsubmission")]
+        return self._fetch_metadata_batch(urls)
+
+    def front_page(
+        self,
+        conduct=Conduct.MOVIES,
+        grouping=Grouping.FEATURED,
+        interval=Interval.TODAY,
+        sort_by=SortBy.DATE_DESC,
+        category="All",
+        offset=0,
+    ):
+        # TODO: Support `after` parameter (e.g. `before=2024-06-04`)
+        # TODO: Support `before` parameter (e.g. `before=2024-06-04`)
+        URLS = {
+            Grouping.FEATURED.value: "featured",
+            Grouping.LATEST.value: "browse",
+            Grouping.POPULAR.value: "popular",
+        }
+
+        assert conduct == Conduct.MOVIES or conduct == Conduct.AUDIO
+        assert isinstance(category, str)
+        assert isinstance(offset, int)
+
+        if conduct == Conduct.MOVIES:
+            assert category in MOVIE_CATEGORIES
+        elif conduct == Conduct.AUDIO:
+            assert category in AUDIO_CATEGORIES
+
+        url = URLS[grouping]
+        response = self._request(
+            "GET",
+            "https://www.newgrounds.com/movies/" + url
+            if conduct == Conduct.MOVIES
+            else "https://www.newgrounds.com/audio/" + url,
+            params={
+                "interval": interval,
+                "sort": stringify_enum(sort_by),
+                "genre": category,
+                "offset": offset,
+                "inner": 1,
+                "isAjaxRequest": True,
+            },
+            headers={
+                "X-Requested-With": "XMLHttpRequest",
+            },
+        ).json()
+
+        soup = BeautifulSoup(response["content"], "html.parser")
+        urls = [
+            tag["href"]
+            for tag in soup.find_all(
+                "a",
+                "inline-card-portalsubmission"
+                if conduct == Conduct.MOVIES
+                else "item-audiosubmission",
+            )
+        ]
+        return self._fetch_metadata_batch(urls)
+
+    def feed(self):
+        html = self._request(
+            "GET",
+            "https://www.newgrounds.com/social/feeds/show/favorite-artists-movies",
+            cookies=get_session(),
+        ).text
+        soup = BeautifulSoup(html, "html.parser")
+        urls = [tag["href"] for tag in soup.find_all("a", "item-portalsubmission")]
+        return self._fetch_metadata_batch(urls)
+
+    def radio_status(self):
+        # https://stream.newgroundsradio.com/radio.mp3
+        pass