diff options
| -rw-r--r-- | doc/API.md | 10 | ||||
| -rw-r--r-- | server/config.yaml.dist | 11 | ||||
| -rw-r--r-- | server/szurubooru/facade.py | 5 | ||||
| -rw-r--r-- | server/szurubooru/func/net.py | 24 | ||||
| -rw-r--r-- | server/szurubooru/func/snapshots.py | 13 | ||||
| -rw-r--r-- | server/szurubooru/tests/api/test_pool_category_updating.py | 2 | ||||
| -rw-r--r-- | server/szurubooru/tests/api/test_pool_creating.py | 2 | ||||
| -rw-r--r-- | server/szurubooru/tests/api/test_pool_deleting.py | 23 | ||||
| -rw-r--r-- | server/szurubooru/tests/api/test_post_creating.py | 12 | ||||
| -rw-r--r-- | server/szurubooru/tests/api/test_post_featuring.py | 8 | ||||
| -rw-r--r-- | server/szurubooru/tests/api/test_tag_category_updating.py | 2 | ||||
| -rw-r--r-- | server/szurubooru/tests/api/test_tag_creating.py | 2 | ||||
| -rw-r--r-- | server/szurubooru/tests/api/test_tag_deleting.py | 21 | ||||
| -rw-r--r-- | server/szurubooru/tests/func/test_net.py | 45 | ||||
| -rw-r--r-- | server/szurubooru/tests/func/test_snapshots.py | 64 |
15 files changed, 182 insertions, 62 deletions
@@ -13,6 +13,7 @@ - [Error handling](#error-handling) - [Field selecting](#field-selecting) - [Versioning](#versioning) + - [Webhooks](#webhooks) 2. [API reference](#api-reference) @@ -276,6 +277,15 @@ reject the request as well, in which case the client is encouraged to notify the user about the situation. +## Webhooks + +System administrators can choose to configure webhooks to track events. +Webhook URIs can be configured in `config.yaml` (See `config.yaml.dist` for +example). Upon any event, the API will send a `POST` request to the listed +URIs with a [snapshot resource](#snapshot) generated with anonymous user +privileges as the message body, in JSON format. + + # API reference Depending on the deployment, the URLs might be relative to some base path such diff --git a/server/config.yaml.dist b/server/config.yaml.dist index 937c138..e3799a3 100644 --- a/server/config.yaml.dist +++ b/server/config.yaml.dist @@ -39,8 +39,9 @@ smtp: user: # example: bot pass: # example: groovy123 from: # example: noreply@example.com - # if host is left empty the password reset feature will be disabled, in which case it is - # recommended to fill contactEmail so that users know who to contact when they want to reset their password + # if host is left empty the password reset feature will be disabled, + # in which case it is recommended to fill contactEmail so that users + # know who to contact when they want to reset their password contact_email: # example: bob@example.com. Meant for manual password reset procedures @@ -58,6 +59,12 @@ pool_category_name_regex: ^[^\s%+#/]+$ password_regex: '^.{5,}$' user_name_regex: '^[a-zA-Z0-9_-]{1,32}$' +# webhooks to call when events occur (such as post/tag/user/etc. changes) +# the listed urls will be called with a HTTP POST request with a payload +# containing a snapshot resource as JSON. See doc/API.md for details +webhooks: + # - https://api.example.com/webhooks/ + default_rank: regular privileges: diff --git a/server/szurubooru/facade.py b/server/szurubooru/facade.py index f1a9a07..ecf34c7 100644 --- a/server/szurubooru/facade.py +++ b/server/szurubooru/facade.py @@ -94,6 +94,11 @@ def validate_config() -> None: if not config.config["database"]: raise errors.ConfigError("Database is not configured") + if config.config["webhooks"] and not isinstance( + config.config["webhooks"], list + ): + raise errors.ConfigError("Webhooks must be provided as a list of URLs") + if config.config["smtp"]["host"]: if not config.config["smtp"]["port"]: raise errors.ConfigError("SMTP host is set but port is not set") diff --git a/server/szurubooru/func/net.py b/server/szurubooru/func/net.py index ddac65c..bb43ddd 100644 --- a/server/szurubooru/func/net.py +++ b/server/szurubooru/func/net.py @@ -1,7 +1,10 @@ +import json import logging import os +import urllib.error import urllib.request from tempfile import NamedTemporaryFile +from typing import Any, Dict, List from youtube_dl import YoutubeDL from youtube_dl.utils import YoutubeDLError @@ -58,3 +61,24 @@ def _youtube_dl_wrapper(url: str) -> bytes: raise errors.ThirdPartyError( "Error downloading video %s (file could not be saved)" % (url) ) + + +def post_to_webhooks(payload: Dict[str, Any]) -> List[int]: + return_list = [] + for webhook in config.config["webhooks"] or []: + req = urllib.request.Request(webhook) + req.data = json.dumps( + payload, default=lambda x: x.isoformat("T") + "Z", + ).encode("utf-8") + req.add_header("Content-Type", "application/json") + try: + res = urllib.request.urlopen(req) + if not 200 <= res.status <= 299: + logger.warning( + f"Webhook {webhook} returned {res.status} {res.reason}" + ) + return_list.append(res.status) + except urllib.error.URLError as e: + logger.error(f"Unable to call webhook {webhook}: {str(e)}") + return_list.append(400) + return return_list diff --git a/server/szurubooru/func/snapshots.py b/server/szurubooru/func/snapshots.py index b34b702..afb26ea 100644 --- a/server/szurubooru/func/snapshots.py +++ b/server/szurubooru/func/snapshots.py @@ -4,7 +4,7 @@ from typing import Any, Callable, Dict, Optional import sqlalchemy as sa from szurubooru import db, model -from szurubooru.func import diff, users +from szurubooru.func import diff, net, users def get_tag_category_snapshot(category: model.TagCategory) -> Dict[str, Any]: @@ -93,6 +93,13 @@ def serialize_snapshot( } +def _post_to_webhooks(snapshot: model.Snapshot) -> None: + webhook_user = model.User() + webhook_user.name = None + webhook_user.rank = "anonymous" + net.post_to_webhooks(serialize_snapshot(snapshot, webhook_user)) + + def _create( operation: str, entity: model.Base, auth_user: Optional[model.User] ) -> model.Snapshot: @@ -116,6 +123,7 @@ def create(entity: model.Base, auth_user: Optional[model.User]) -> None: snapshot_factory = _snapshot_factories[snapshot.resource_type] snapshot.data = snapshot_factory(entity) db.session.add(snapshot) + _post_to_webhooks(snapshot) def modify(entity: model.Base, auth_user: Optional[model.User]) -> None: @@ -147,6 +155,7 @@ def modify(entity: model.Base, auth_user: Optional[model.User]) -> None: if not snapshot.data: return db.session.add(snapshot) + _post_to_webhooks(snapshot) def delete(entity: model.Base, auth_user: Optional[model.User]) -> None: @@ -155,6 +164,7 @@ def delete(entity: model.Base, auth_user: Optional[model.User]) -> None: snapshot_factory = _snapshot_factories[snapshot.resource_type] snapshot.data = snapshot_factory(entity) db.session.add(snapshot) + _post_to_webhooks(snapshot) def merge( @@ -174,3 +184,4 @@ def merge( ) = model.util.get_resource_info(target_entity) snapshot.data = [resource_type, resource_name] db.session.add(snapshot) + _post_to_webhooks(snapshot) diff --git a/server/szurubooru/tests/api/test_pool_category_updating.py b/server/szurubooru/tests/api/test_pool_category_updating.py index d934e60..c13112f 100644 --- a/server/szurubooru/tests/api/test_pool_category_updating.py +++ b/server/szurubooru/tests/api/test_pool_category_updating.py @@ -67,7 +67,7 @@ def test_omitting_optional_field( del params[field] with patch("szurubooru.func.pool_categories.serialize_category"), patch( "szurubooru.func.pool_categories.update_category_name" - ): + ), patch("szurubooru.func.snapshots._post_to_webhooks"): api.pool_category_api.update_pool_category( context_factory( params={**params, **{"version": 1}}, diff --git a/server/szurubooru/tests/api/test_pool_creating.py b/server/szurubooru/tests/api/test_pool_creating.py index a9f1473..4fd8939 100644 --- a/server/szurubooru/tests/api/test_pool_creating.py +++ b/server/szurubooru/tests/api/test_pool_creating.py @@ -70,7 +70,7 @@ def test_omitting_optional_field( del params[field] with patch("szurubooru.func.pools.create_pool"), patch( "szurubooru.func.pools.serialize_pool" - ): + ), patch("szurubooru.func.snapshots._post_to_webhooks"): pools.create_pool.return_value = pool_factory() api.pool_api.create_pool( context_factory( diff --git a/server/szurubooru/tests/api/test_pool_deleting.py b/server/szurubooru/tests/api/test_pool_deleting.py index 387aca1..5c5adcc 100644 --- a/server/szurubooru/tests/api/test_pool_deleting.py +++ b/server/szurubooru/tests/api/test_pool_deleting.py @@ -34,17 +34,18 @@ def test_deleting_used( pool.posts.append(post) db.session.add_all([pool, post]) db.session.commit() - api.pool_api.delete_pool( - context_factory( - params={"version": 1}, - user=user_factory(rank=model.User.RANK_REGULAR), - ), - {"pool_id": 1}, - ) - db.session.refresh(post) - assert db.session.query(model.Pool).count() == 0 - assert db.session.query(model.PoolPost).count() == 0 - assert post.pools == [] + with patch("szurubooru.func.snapshots._post_to_webhooks"): + api.pool_api.delete_pool( + context_factory( + params={"version": 1}, + user=user_factory(rank=model.User.RANK_REGULAR), + ), + {"pool_id": 1}, + ) + db.session.refresh(post) + assert db.session.query(model.Pool).count() == 0 + assert db.session.query(model.PoolPost).count() == 0 + assert post.pools == [] def test_trying_to_delete_non_existing(user_factory, context_factory): diff --git a/server/szurubooru/tests/api/test_post_creating.py b/server/szurubooru/tests/api/test_post_creating.py index 4e05cf1..e2db0eb 100644 --- a/server/szurubooru/tests/api/test_post_creating.py +++ b/server/szurubooru/tests/api/test_post_creating.py @@ -143,7 +143,9 @@ def test_anonymous_uploads( with patch("szurubooru.func.posts.serialize_post"), patch( "szurubooru.func.posts.create_post" - ), patch("szurubooru.func.posts.update_post_source"): + ), patch("szurubooru.func.posts.update_post_source"), patch( + "szurubooru.func.snapshots._post_to_webhooks" + ): config_injector( { "privileges": { @@ -181,6 +183,8 @@ def test_creating_from_url_saves_source( "szurubooru.func.posts.serialize_post" ), patch("szurubooru.func.posts.create_post"), patch( "szurubooru.func.posts.update_post_source" + ), patch( + "szurubooru.func.snapshots._post_to_webhooks" ): config_injector( { @@ -223,6 +227,8 @@ def test_creating_from_url_with_source_specified( "szurubooru.func.posts.serialize_post" ), patch("szurubooru.func.posts.create_post"), patch( "szurubooru.func.posts.update_post_source" + ), patch( + "szurubooru.func.snapshots._post_to_webhooks" ): config_injector( { @@ -334,7 +340,7 @@ def test_errors_not_spending_ids( # successful request with patch("szurubooru.func.posts.serialize_post"), patch( "szurubooru.func.posts.update_post_tags" - ): + ), patch("szurubooru.func.snapshots._post_to_webhooks"): posts.serialize_post.side_effect = lambda post, *_, **__: post.post_id post1_id = api.post_api.create_post( context_factory( @@ -357,7 +363,7 @@ def test_errors_not_spending_ids( # successful request with patch("szurubooru.func.posts.serialize_post"), patch( "szurubooru.func.posts.update_post_tags" - ): + ), patch("szurubooru.func.snapshots._post_to_webhooks"): posts.serialize_post.side_effect = lambda post, *_, **__: post.post_id post2_id = api.post_api.create_post( context_factory( diff --git a/server/szurubooru/tests/api/test_post_featuring.py b/server/szurubooru/tests/api/test_post_featuring.py index e1f8cef..d83d4b6 100644 --- a/server/szurubooru/tests/api/test_post_featuring.py +++ b/server/szurubooru/tests/api/test_post_featuring.py @@ -55,7 +55,9 @@ def test_trying_to_feature_the_same_post_twice( ): db.session.add(post_factory(id=1)) db.session.commit() - with patch("szurubooru.func.posts.serialize_post"): + with patch("szurubooru.func.posts.serialize_post"), patch( + "szurubooru.func.snapshots._post_to_webhooks" + ): api.post_api.set_featured_post( context_factory( params={"id": 1}, @@ -80,7 +82,9 @@ def test_featuring_one_post_after_another( assert posts.try_get_featured_post() is None assert not posts.get_post_by_id(1).is_featured assert not posts.get_post_by_id(2).is_featured - with patch("szurubooru.func.posts.serialize_post"): + with patch("szurubooru.func.posts.serialize_post"), patch( + "szurubooru.func.snapshots._post_to_webhooks" + ): with fake_datetime("1997"): api.post_api.set_featured_post( context_factory( diff --git a/server/szurubooru/tests/api/test_tag_category_updating.py b/server/szurubooru/tests/api/test_tag_category_updating.py index 0d59390..6c32737 100644 --- a/server/szurubooru/tests/api/test_tag_category_updating.py +++ b/server/szurubooru/tests/api/test_tag_category_updating.py @@ -65,7 +65,7 @@ def test_omitting_optional_field( del params[field] with patch("szurubooru.func.tag_categories.serialize_category"), patch( "szurubooru.func.tag_categories.update_category_name" - ): + ), patch("szurubooru.func.snapshots._post_to_webhooks"): api.tag_category_api.update_tag_category( context_factory( params={**params, **{"version": 1}}, diff --git a/server/szurubooru/tests/api/test_tag_creating.py b/server/szurubooru/tests/api/test_tag_creating.py index 648b317..2f4264d 100644 --- a/server/szurubooru/tests/api/test_tag_creating.py +++ b/server/szurubooru/tests/api/test_tag_creating.py @@ -71,7 +71,7 @@ def test_omitting_optional_field( del params[field] with patch("szurubooru.func.tags.create_tag"), patch( "szurubooru.func.tags.serialize_tag" - ): + ), patch("szurubooru.func.snapshots._post_to_webhooks"): tags.create_tag.return_value = tag_factory() api.tag_api.create_tag( context_factory( diff --git a/server/szurubooru/tests/api/test_tag_deleting.py b/server/szurubooru/tests/api/test_tag_deleting.py index b45ef55..59a19f8 100644 --- a/server/szurubooru/tests/api/test_tag_deleting.py +++ b/server/szurubooru/tests/api/test_tag_deleting.py @@ -34,16 +34,17 @@ def test_deleting_used( post.tags.append(tag) db.session.add_all([tag, post]) db.session.commit() - api.tag_api.delete_tag( - context_factory( - params={"version": 1}, - user=user_factory(rank=model.User.RANK_REGULAR), - ), - {"tag_name": "tag"}, - ) - db.session.refresh(post) - assert db.session.query(model.Tag).count() == 0 - assert post.tags == [] + with patch("szurubooru.func.snapshots._post_to_webhooks"): + api.tag_api.delete_tag( + context_factory( + params={"version": 1}, + user=user_factory(rank=model.User.RANK_REGULAR), + ), + {"tag_name": "tag"}, + ) + db.session.refresh(post) + assert db.session.query(model.Tag).count() == 0 + assert post.tags == [] def test_trying_to_delete_non_existing(user_factory, context_factory): diff --git a/server/szurubooru/tests/func/test_net.py b/server/szurubooru/tests/func/test_net.py index 58e1628..8539a34 100644 --- a/server/szurubooru/tests/func/test_net.py +++ b/server/szurubooru/tests/func/test_net.py @@ -1,3 +1,6 @@ +from datetime import datetime +from unittest.mock import patch + import pytest from szurubooru import errors @@ -99,3 +102,45 @@ def test_video_download(url, expected_sha1): def test_failed_video_download(url): with pytest.raises(errors.ThirdPartyError): net.download(url, use_video_downloader=True) + + +def test_no_webhooks(config_injector): + config_injector({"webhooks": []}) + res = net.post_to_webhooks(None) + assert len(res) == 0 + + +@pytest.mark.parametrize( + "webhook,status_code", + [ + ("https://postman-echo.com/post", 200), + ("http://localhost/", 400), + ("https://postman-echo.com/get", 400), + ], +) +def test_single_webhook(config_injector, webhook, status_code): + config_injector({"webhooks": [webhook]}) + res = net.post_to_webhooks({"test_arg": "test_value"}) + assert len(res) == 1 + assert res[0] == status_code + + +def test_multiple_webhooks(config_injector): + config_injector( + { + "webhooks": [ + "https://postman-echo.com/post", + "https://postman-echo.com/post", + ] + } + ) + res = net.post_to_webhooks({"test_arg": "test_value"}) + assert len(res) == 2 + assert res[0] == 200 + assert res[1] == 200 + + +def test_malformed_webhooks(config_injector): + config_injector({"webhooks": ["malformed_url"]}) + with pytest.raises(ValueError): + net.post_to_webhooks({"test_arg": "test_value"}) diff --git a/server/szurubooru/tests/func/test_snapshots.py b/server/szurubooru/tests/func/test_snapshots.py index b4dda1c..da93530 100644 --- a/server/szurubooru/tests/func/test_snapshots.py +++ b/server/szurubooru/tests/func/test_snapshots.py @@ -132,7 +132,9 @@ def test_create(tag_factory, user_factory): tag = tag_factory(names=["dummy"]) db.session.add(tag) db.session.flush() - with patch("szurubooru.func.snapshots.get_tag_snapshot"): + with patch("szurubooru.func.snapshots.get_tag_snapshot"), patch( + "szurubooru.func.snapshots._post_to_webhooks" + ): snapshots.get_tag_snapshot.return_value = "mocked" snapshots.create(tag, user_factory()) db.session.flush() @@ -156,29 +158,30 @@ def test_modify_saves_non_empty_diffs(post_factory, user_factory): post.source = "new source" post.notes = [model.PostNote(polygon=[(0, 0), (0, 1), (1, 1)], text="new")] db.session.flush() - snapshots.modify(post, user) - db.session.flush() - results = db.session.query(model.Snapshot).all() - assert len(results) == 1 - assert results[0].data == { - "type": "object change", - "value": { - "source": { - "type": "primitive change", - "old-value": None, - "new-value": "new source", - }, - "notes": { - "type": "list change", - "removed": [ - {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"} - ], - "added": [ - {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"} - ], + with patch("szurubooru.func.snapshots._post_to_webhooks"): + snapshots.modify(post, user) + db.session.flush() + results = db.session.query(model.Snapshot).all() + assert len(results) == 1 + assert results[0].data == { + "type": "object change", + "value": { + "source": { + "type": "primitive change", + "old-value": None, + "new-value": "new source", + }, + "notes": { + "type": "list change", + "removed": [ + {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "old"} + ], + "added": [ + {"polygon": [[0, 0], [0, 1], [1, 1]], "text": "new"} + ], + }, }, - }, - } + } def test_modify_doesnt_save_empty_diffs(tag_factory, user_factory): @@ -195,7 +198,9 @@ def test_delete(tag_factory, user_factory): tag = tag_factory(names=["dummy"]) db.session.add(tag) db.session.flush() - with patch("szurubooru.func.snapshots.get_tag_snapshot"): + with patch("szurubooru.func.snapshots.get_tag_snapshot"), patch( + "szurubooru.func.snapshots._post_to_webhooks" + ): snapshots.get_tag_snapshot.return_value = "mocked" snapshots.delete(tag, user_factory()) db.session.flush() @@ -210,8 +215,9 @@ def test_merge(tag_factory, user_factory): target_tag = tag_factory(names=["target"]) db.session.add_all([source_tag, target_tag]) db.session.flush() - snapshots.merge(source_tag, target_tag, user_factory()) - db.session.flush() - result = db.session.query(model.Snapshot).one() - assert result.operation == model.Snapshot.OPERATION_MERGED - assert result.data == ["tag", "target"] + with patch("szurubooru.func.snapshots._post_to_webhooks"): + snapshots.merge(source_tag, target_tag, user_factory()) + db.session.flush() + result = db.session.query(model.Snapshot).one() + assert result.operation == model.Snapshot.OPERATION_MERGED + assert result.data == ["tag", "target"] |