From 32d95db949af732e4dd43e26e83c311e1b107fef Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Thu, 6 Jun 2024 15:44:11 +0200
Subject: [PATCH 01/17] add: first meta
---
djangoldp/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/djangoldp/__init__.py b/djangoldp/__init__.py
index efc86af2..a99e40eb 100644
--- a/djangoldp/__init__.py
+++ b/djangoldp/__init__.py
@@ -5,4 +5,4 @@ __version__ = '0.0.0'
options.DEFAULT_NAMES += (
'lookup_field', 'rdf_type', 'rdf_context', 'auto_author', 'owner_field', 'owner_urlid_field',
'view_set', 'container_path', 'permission_classes', 'serializer_fields', 'serializer_fields_exclude', 'empty_containers',
- 'nested_fields', 'depth', 'permission_roles', 'inherit_permissions', 'public_field')
+ 'nested_fields', 'depth', 'permission_roles', 'inherit_permissions', 'public_field', 'static_version')
--
GitLab
From 69d0b962866061e3eaf32050021c4143af6854bd Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 7 Jun 2024 11:25:47 +0200
Subject: [PATCH 02/17] feature: generate models static version by command
---
djangoldp/__init__.py | 2 +-
.../commands/generate_static_content.py | 74 +++++++++++++++++++
djangoldp/urls.py | 6 +-
djangoldp/views.py | 20 ++++-
4 files changed, 98 insertions(+), 4 deletions(-)
create mode 100644 djangoldp/management/commands/generate_static_content.py
diff --git a/djangoldp/__init__.py b/djangoldp/__init__.py
index a99e40eb..7f7595fa 100644
--- a/djangoldp/__init__.py
+++ b/djangoldp/__init__.py
@@ -5,4 +5,4 @@ __version__ = '0.0.0'
options.DEFAULT_NAMES += (
'lookup_field', 'rdf_type', 'rdf_context', 'auto_author', 'owner_field', 'owner_urlid_field',
'view_set', 'container_path', 'permission_classes', 'serializer_fields', 'serializer_fields_exclude', 'empty_containers',
- 'nested_fields', 'depth', 'permission_roles', 'inherit_permissions', 'public_field', 'static_version')
+ 'nested_fields', 'depth', 'permission_roles', 'inherit_permissions', 'public_field', 'static_version', 'static_params')
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
new file mode 100644
index 00000000..667ab55c
--- /dev/null
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -0,0 +1,74 @@
+import os
+import requests
+import json
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.apps import apps
+from urllib.parse import urlparse, urlunparse
+
+class Command(BaseCommand):
+ help = 'Generate static content for models with a specific meta attribute'
+
+ def handle(self, *args, **kwargs):
+ output_dir = 'ssr'
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir, exist_ok=True)
+
+ base_uri = getattr(settings, 'BASE_URL', '')
+
+ for model in apps.get_models():
+ if hasattr(model._meta, 'static_version'):
+ print(f"model: {model}")
+ container_path = model.get_container_path()
+ url = f'{base_uri}{container_path}'
+ print(f"current request url before adding params: {url}")
+
+ if hasattr(model._meta, 'static_params'):
+ # static_params which is a json must be decomposed and added to the url as query parameters, first with ? then with &
+ url += '?'
+ for key, value in model._meta.static_params.items():
+ url += f'{key}={value}&'
+ url = url[:-1]
+
+ print(f"current request url after adding params: {url}")
+ response = requests.get(url)
+
+ if response.status_code == 200:
+ content = response.text
+ content = self.update_ids(content, base_uri)
+
+ filename = container_path[1:-1]
+ file_path = os.path.join(output_dir, f'{filename}.json')
+
+ print(f"file_path: {file_path}")
+ with open(file_path, 'w') as f:
+ f.write(content)
+ self.stdout.write(self.style.SUCCESS(f'Successfully fetched and saved content for {model._meta.model_name} from {url}'))
+ else:
+ self.stdout.write(self.style.ERROR(f'Failed to fetch content from {url}: {response.status_code}'))
+
+ def update_ids(self, content, base_uri):
+ try:
+ data = json.loads(content)
+ if isinstance(data, list):
+ for item in data:
+ self.update_item_id(item, base_uri)
+ elif isinstance(data, dict):
+ self.update_item_id(data, base_uri)
+ return json.dumps(data)
+ except json.JSONDecodeError as e:
+ self.stdout.write(self.style.ERROR(f'Failed to decode JSON: {e}'))
+ return content
+
+ def update_item_id(self, item, base_uri):
+ if '@id' in item:
+ parsed_url = urlparse(item['@id'])
+ path = f'/ssr{parsed_url.path}'
+ item['@id'] = urlunparse((parsed_url.scheme, parsed_url.netloc, path, parsed_url.params, parsed_url.query, parsed_url.fragment))
+ for key, value in item.items():
+ if isinstance(value, dict):
+ self.update_item_id(value, base_uri)
+ elif isinstance(value, list):
+ for sub_item in value:
+ if isinstance(sub_item, dict):
+ self.update_item_id(sub_item, base_uri)
\ No newline at end of file
diff --git a/djangoldp/urls.py b/djangoldp/urls.py
index 86682c5f..1cc3f17f 100644
--- a/djangoldp/urls.py
+++ b/djangoldp/urls.py
@@ -7,7 +7,8 @@ from django.urls import path, re_path, include
from djangoldp.models import LDPSource, Model
from djangoldp.permissions import ReadOnly
from djangoldp.views import LDPSourceViewSet, WebFingerView, InboxView
-from djangoldp.views import LDPViewSet
+from djangoldp.views import LDPViewSet, serve_static_content
+
def __clean_path(path):
'''ensures path is Django-friendly'''
@@ -35,7 +36,8 @@ urlpatterns = [
re_path(r'^sources/(?P<federation>\w+)/', LDPSourceViewSet.urls(model=LDPSource, fields=['federation', 'urlid'],
permission_classes=[ReadOnly], )),
re_path(r'^\.well-known/webfinger/?$', WebFingerView.as_view()),
- path('inbox/', InboxView.as_view())
+ path('inbox/', InboxView.as_view()),
+ path('ssr/<str:filename>/', serve_static_content, name='serve_static_content'),
]
if settings.ENABLE_SWAGGER_DOCUMENTATION:
diff --git a/djangoldp/views.py b/djangoldp/views.py
index 27f0400a..fa5cf7d4 100644
--- a/djangoldp/views.py
+++ b/djangoldp/views.py
@@ -5,7 +5,7 @@ from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist
from django.db import IntegrityError, transaction
-from django.http import JsonResponse, Http404
+from django.http import JsonResponse, Http404, HttpResponseNotFound
from django.shortcuts import get_object_or_404
from django.urls import include, re_path, path
from django.urls.resolvers import get_resolver
@@ -31,6 +31,7 @@ from djangoldp.utils import is_authenticated_user
from djangoldp.activities import ActivityQueueService, as_activitystream, ACTIVITY_SAVING_SETTING, ActivityPubService
from djangoldp.activities.errors import ActivityStreamDecodeError, ActivityStreamValidationError
import logging
+import os
logger = logging.getLogger('djangoldp')
get_user_model()._meta.rdf_context = {"get_full_name": "rdfs:label"}
@@ -615,3 +616,20 @@ class WebFingerView(View):
def post(self, request, *args, **kwargs):
return self.on_request(request)
+
+
+def serve_static_content(request, filename):
+ file_path = os.path.join('ssr', filename)
+ # Transform file_path to valid filename by adding .json extension
+ if not file_path.endswith('.json'):
+ file_path += '.json'
+
+ print(f"file_path: {file_path}")
+ if os.path.exists(file_path):
+ with open(file_path, 'r') as file:
+ content = file.read()
+
+ json_content = json.loads(content)
+ return JsonResponse(json_content, safe=False, status=200, content_type='application/ld+json')
+ else:
+ return HttpResponseNotFound('File not found')
--
GitLab
From cb55310d4d3a87294590370ae825fccc1614be30 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 7 Jun 2024 11:53:44 +0200
Subject: [PATCH 03/17] push: fix
---
djangoldp/views.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/djangoldp/views.py b/djangoldp/views.py
index fa5cf7d4..013c025d 100644
--- a/djangoldp/views.py
+++ b/djangoldp/views.py
@@ -630,6 +630,6 @@ def serve_static_content(request, filename):
content = file.read()
json_content = json.loads(content)
- return JsonResponse(json_content, safe=False, status=200, content_type='application/ld+json')
+ return JsonResponse(json_content, safe=False, status=200, content_type='application/ld+json', headers={'Access-Control-Allow-Origin': '*'})
else:
return HttpResponseNotFound('File not found')
--
GitLab
From 781acff75c8f46a54fa0734c26c89c8580b70b1e Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 7 Jun 2024 12:06:21 +0200
Subject: [PATCH 04/17] fix: simplify
---
.../commands/generate_static_content.py | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index 667ab55c..fb83d8ef 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -29,13 +29,13 @@ class Command(BaseCommand):
for key, value in model._meta.static_params.items():
url += f'{key}={value}&'
url = url[:-1]
-
+
print(f"current request url after adding params: {url}")
response = requests.get(url)
if response.status_code == 200:
content = response.text
- content = self.update_ids(content, base_uri)
+ content = self.update_ids(content, base_uri, model._meta.model_name.lower())
filename = container_path[1:-1]
file_path = os.path.join(output_dir, f'{filename}.json')
@@ -47,28 +47,28 @@ class Command(BaseCommand):
else:
self.stdout.write(self.style.ERROR(f'Failed to fetch content from {url}: {response.status_code}'))
- def update_ids(self, content, base_uri):
+ def update_ids(self, content, base_uri, model_name):
try:
data = json.loads(content)
if isinstance(data, list):
for item in data:
- self.update_item_id(item, base_uri)
+ self.update_item_id(item, base_uri, model_name)
elif isinstance(data, dict):
- self.update_item_id(data, base_uri)
+ self.update_item_id(data, base_uri, model_name)
return json.dumps(data)
except json.JSONDecodeError as e:
self.stdout.write(self.style.ERROR(f'Failed to decode JSON: {e}'))
return content
- def update_item_id(self, item, base_uri):
- if '@id' in item:
+ def update_item_id(self, item, base_uri, model_name):
+ if '@id' in item and model_name in item['@id']:
parsed_url = urlparse(item['@id'])
path = f'/ssr{parsed_url.path}'
item['@id'] = urlunparse((parsed_url.scheme, parsed_url.netloc, path, parsed_url.params, parsed_url.query, parsed_url.fragment))
for key, value in item.items():
if isinstance(value, dict):
- self.update_item_id(value, base_uri)
+ self.update_item_id(value, base_uri, model_name)
elif isinstance(value, list):
for sub_item in value:
if isinstance(sub_item, dict):
- self.update_item_id(sub_item, base_uri)
\ No newline at end of file
+ self.update_item_id(sub_item, base_uri, model_name)
\ No newline at end of file
--
GitLab
From b00f39891fe83a6539403445a1cfe2340954505c Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 7 Jun 2024 12:21:07 +0200
Subject: [PATCH 05/17] revert: prev
---
djangoldp/management/commands/generate_static_content.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index fb83d8ef..6e42515f 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -61,7 +61,7 @@ class Command(BaseCommand):
return content
def update_item_id(self, item, base_uri, model_name):
- if '@id' in item and model_name in item['@id']:
+ if '@id' in item:
parsed_url = urlparse(item['@id'])
path = f'/ssr{parsed_url.path}'
item['@id'] = urlunparse((parsed_url.scheme, parsed_url.netloc, path, parsed_url.params, parsed_url.query, parsed_url.fragment))
--
GitLab
From 88ee5842b21cbf912dcfa99bdb234fcc292994f6 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 7 Jun 2024 17:42:22 +0200
Subject: [PATCH 06/17] fix: proper recursion support and usage of paths
---
.../commands/generate_static_content.py | 46 +++++++++++++++----
djangoldp/urls.py | 3 +-
djangoldp/views.py | 4 +-
3 files changed, 42 insertions(+), 11 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index 6e42515f..1f032fb9 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -15,6 +15,7 @@ class Command(BaseCommand):
os.makedirs(output_dir, exist_ok=True)
base_uri = getattr(settings, 'BASE_URL', '')
+ max_depth = getattr(settings, 'MAX_RECURSION_DEPTH', 3) # Ad
for model in apps.get_models():
if hasattr(model._meta, 'static_version'):
@@ -31,11 +32,11 @@ class Command(BaseCommand):
url = url[:-1]
print(f"current request url after adding params: {url}")
- response = requests.get(url)
+ response = requests.get(url, timeout=settings.SSR_REQUEST_TIMEOUT)
if response.status_code == 200:
content = response.text
- content = self.update_ids(content, base_uri, model._meta.model_name.lower())
+ content = self.update_ids_and_fetch_associated(content, base_uri, output_dir, 0, max_depth)
filename = container_path[1:-1]
file_path = os.path.join(output_dir, f'{filename}.json')
@@ -47,28 +48,57 @@ class Command(BaseCommand):
else:
self.stdout.write(self.style.ERROR(f'Failed to fetch content from {url}: {response.status_code}'))
- def update_ids(self, content, base_uri, model_name):
+ def update_ids_and_fetch_associated(self, content, base_uri, output_dir, depth, max_depth):
+ if depth > max_depth:
+ return content
+
try:
data = json.loads(content)
if isinstance(data, list):
for item in data:
- self.update_item_id(item, base_uri, model_name)
+ self.update_and_fetch_id(item, base_uri, output_dir, depth, max_depth)
elif isinstance(data, dict):
- self.update_item_id(data, base_uri, model_name)
+ self.update_and_fetch_id(data, base_uri, output_dir, depth, max_depth)
return json.dumps(data)
except json.JSONDecodeError as e:
self.stdout.write(self.style.ERROR(f'Failed to decode JSON: {e}'))
return content
- def update_item_id(self, item, base_uri, model_name):
+ def update_and_fetch_id(self, item, base_uri, output_dir, depth, max_depth):
if '@id' in item:
parsed_url = urlparse(item['@id'])
path = f'/ssr{parsed_url.path}'
item['@id'] = urlunparse((parsed_url.scheme, parsed_url.netloc, path, parsed_url.params, parsed_url.query, parsed_url.fragment))
+ associated_url = urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment))
+
+ # Fetch associated content
+ try:
+ response = requests.get(associated_url, timeout=settings.SSR_REQUEST_TIMEOUT)
+ if response.status_code == 200:
+ associated_content = response.text
+ associated_content = self.update_ids_and_fetch_associated(associated_content, base_uri, output_dir, depth + 1, max_depth)
+ # associated_content = self.update_ids_and_fetch_associated(response.text, base_uri, output_dir)
+ # associated_content = self.update_and_fetch_id(response.text, base_uri, output_dir)
+ associated_file_path = path[1:-1] + '.json'
+ print(f"associated_file_path: {associated_file_path}")
+ associated_file_dir = os.path.dirname(associated_file_path)
+ print(f"associated_file_path: {associated_file_dir}")
+ if not os.path.exists(associated_file_dir):
+ os.makedirs(associated_file_dir)
+ with open(associated_file_path, 'w') as f:
+ f.write(associated_content)
+ self.stdout.write(self.style.SUCCESS(f'Successfully fetched and saved associated content for {associated_url}'))
+ else:
+ self.stdout.write(self.style.ERROR(f'Failed to fetch associated content from {associated_url}: {response.status_code}'))
+ except requests.exceptions.Timeout:
+ self.stdout.write(self.style.ERROR(f'Request to {associated_url} timed out'))
+ except requests.exceptions.RequestException as e:
+ self.stdout.write(self.style.ERROR(f'An error occurred: {e}'))
+
for key, value in item.items():
if isinstance(value, dict):
- self.update_item_id(value, base_uri, model_name)
+ self.update_and_fetch_id(value, base_uri, output_dir, depth, max_depth)
elif isinstance(value, list):
for sub_item in value:
if isinstance(sub_item, dict):
- self.update_item_id(sub_item, base_uri, model_name)
\ No newline at end of file
+ self.update_and_fetch_id(sub_item, base_uri, output_dir, depth, max_depth)
\ No newline at end of file
diff --git a/djangoldp/urls.py b/djangoldp/urls.py
index 1cc3f17f..075dddec 100644
--- a/djangoldp/urls.py
+++ b/djangoldp/urls.py
@@ -37,7 +37,8 @@ urlpatterns = [
permission_classes=[ReadOnly], )),
re_path(r'^\.well-known/webfinger/?$', WebFingerView.as_view()),
path('inbox/', InboxView.as_view()),
- path('ssr/<str:filename>/', serve_static_content, name='serve_static_content'),
+ # path('ssr/<str:filename>/', serve_static_content, name='serve_static_content'),
+ re_path(r'^ssr/(?P<path>.*)$', serve_static_content, name='serve_static_content'),
]
if settings.ENABLE_SWAGGER_DOCUMENTATION:
diff --git a/djangoldp/views.py b/djangoldp/views.py
index 013c025d..867a658a 100644
--- a/djangoldp/views.py
+++ b/djangoldp/views.py
@@ -618,8 +618,8 @@ class WebFingerView(View):
return self.on_request(request)
-def serve_static_content(request, filename):
- file_path = os.path.join('ssr', filename)
+def serve_static_content(request, path):
+ file_path = os.path.join('ssr', path[:-1])
# Transform file_path to valid filename by adding .json extension
if not file_path.endswith('.json'):
file_path += '.json'
--
GitLab
From f055b560dd41dcf3c26376e9f1919d5091b2a034 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Mon, 10 Jun 2024 10:51:37 +0200
Subject: [PATCH 07/17] fix: ids update
---
.../commands/generate_static_content.py | 56 ++++++++++---------
1 file changed, 31 insertions(+), 25 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index 1f032fb9..3b9151b6 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -15,7 +15,7 @@ class Command(BaseCommand):
os.makedirs(output_dir, exist_ok=True)
base_uri = getattr(settings, 'BASE_URL', '')
- max_depth = getattr(settings, 'MAX_RECURSION_DEPTH', 3) # Ad
+ max_depth = getattr(settings, 'MAX_RECURSION_DEPTH', 5) # Ad
for model in apps.get_models():
if hasattr(model._meta, 'static_version'):
@@ -57,8 +57,10 @@ class Command(BaseCommand):
if isinstance(data, list):
for item in data:
self.update_and_fetch_id(item, base_uri, output_dir, depth, max_depth)
- elif isinstance(data, dict):
+ else:
self.update_and_fetch_id(data, base_uri, output_dir, depth, max_depth)
+
+ print(f"Content: {data}")
return json.dumps(data)
except json.JSONDecodeError as e:
self.stdout.write(self.style.ERROR(f'Failed to decode JSON: {e}'))
@@ -70,30 +72,34 @@ class Command(BaseCommand):
path = f'/ssr{parsed_url.path}'
item['@id'] = urlunparse((parsed_url.scheme, parsed_url.netloc, path, parsed_url.params, parsed_url.query, parsed_url.fragment))
associated_url = urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment))
+ associated_file_path = path[1:-1] + '.json'
+ print(f"associated_file_path: {associated_file_path}")
+ associated_file_dir = os.path.dirname(associated_file_path)
+ if not os.path.exists(associated_file_dir):
+ os.makedirs(associated_file_dir)
- # Fetch associated content
- try:
- response = requests.get(associated_url, timeout=settings.SSR_REQUEST_TIMEOUT)
- if response.status_code == 200:
- associated_content = response.text
- associated_content = self.update_ids_and_fetch_associated(associated_content, base_uri, output_dir, depth + 1, max_depth)
- # associated_content = self.update_ids_and_fetch_associated(response.text, base_uri, output_dir)
- # associated_content = self.update_and_fetch_id(response.text, base_uri, output_dir)
- associated_file_path = path[1:-1] + '.json'
- print(f"associated_file_path: {associated_file_path}")
- associated_file_dir = os.path.dirname(associated_file_path)
- print(f"associated_file_path: {associated_file_dir}")
- if not os.path.exists(associated_file_dir):
- os.makedirs(associated_file_dir)
- with open(associated_file_path, 'w') as f:
- f.write(associated_content)
- self.stdout.write(self.style.SUCCESS(f'Successfully fetched and saved associated content for {associated_url}'))
- else:
- self.stdout.write(self.style.ERROR(f'Failed to fetch associated content from {associated_url}: {response.status_code}'))
- except requests.exceptions.Timeout:
- self.stdout.write(self.style.ERROR(f'Request to {associated_url} timed out'))
- except requests.exceptions.RequestException as e:
- self.stdout.write(self.style.ERROR(f'An error occurred: {e}'))
+ if not os.path.exists(associated_file_path):
+ # Fetch associated content
+ try:
+ response = requests.get(associated_url, timeout=settings.SSR_REQUEST_TIMEOUT)
+ if response.status_code == 200:
+ associated_content = self.update_ids_and_fetch_associated(response.text, base_uri, output_dir, depth + 1, max_depth)
+ print(f"associated_file_path: {associated_file_path}")
+ associated_file_dir = os.path.dirname(associated_file_path)
+ print(f"associated_file_path: {associated_file_dir}")
+ if not os.path.exists(associated_file_dir):
+ os.makedirs(associated_file_dir)
+ with open(associated_file_path, 'w') as f:
+ f.write(associated_content)
+ self.stdout.write(self.style.SUCCESS(f'Successfully fetched and saved associated content for {associated_url}'))
+ else:
+ self.stdout.write(self.style.ERROR(f'Failed to fetch associated content from {associated_url}: {response.status_code}'))
+ except requests.exceptions.Timeout:
+ self.stdout.write(self.style.ERROR(f'Request to {associated_url} timed out'))
+ except requests.exceptions.RequestException as e:
+ self.stdout.write(self.style.ERROR(f'An error occurred: {e}'))
+ else:
+ self.stdout.write(self.style.SUCCESS(f'Associated file already exists for {associated_url}, skipping fetch'))
for key, value in item.items():
if isinstance(value, dict):
--
GitLab
From 09491ed51e28787d8f1c0245eb479f9fbf1819ed Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Mon, 10 Jun 2024 10:58:39 +0200
Subject: [PATCH 08/17] fix: timeout settings
---
djangoldp/management/commands/generate_static_content.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index 3b9151b6..a91ed244 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -15,7 +15,8 @@ class Command(BaseCommand):
os.makedirs(output_dir, exist_ok=True)
base_uri = getattr(settings, 'BASE_URL', '')
- max_depth = getattr(settings, 'MAX_RECURSION_DEPTH', 5) # Ad
+ max_depth = getattr(settings, 'MAX_RECURSION_DEPTH', 5)
+ request_timeout = getattr(settings, 'SSR_REQUEST_TIMEOUT', 10)
for model in apps.get_models():
if hasattr(model._meta, 'static_version'):
@@ -32,7 +33,7 @@ class Command(BaseCommand):
url = url[:-1]
print(f"current request url after adding params: {url}")
- response = requests.get(url, timeout=settings.SSR_REQUEST_TIMEOUT)
+ response = requests.get(url, timeout=request_timeout)
if response.status_code == 200:
content = response.text
@@ -81,7 +82,7 @@ class Command(BaseCommand):
if not os.path.exists(associated_file_path):
# Fetch associated content
try:
- response = requests.get(associated_url, timeout=settings.SSR_REQUEST_TIMEOUT)
+ response = requests.get(associated_url, timeout=request_timeout)
if response.status_code == 200:
associated_content = self.update_ids_and_fetch_associated(response.text, base_uri, output_dir, depth + 1, max_depth)
print(f"associated_file_path: {associated_file_path}")
--
GitLab
From 7d243012adfab240937202145fd9b94fd9777eb6 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Mon, 10 Jun 2024 11:00:25 +0200
Subject: [PATCH 09/17] fix: prev
---
djangoldp/management/commands/generate_static_content.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index a91ed244..7b553797 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -6,6 +6,10 @@ from django.conf import settings
from django.apps import apps
from urllib.parse import urlparse, urlunparse
+base_uri = getattr(settings, 'BASE_URL', '')
+max_depth = getattr(settings, 'MAX_RECURSION_DEPTH', 5)
+request_timeout = getattr(settings, 'SSR_REQUEST_TIMEOUT', 10)
+
class Command(BaseCommand):
help = 'Generate static content for models with a specific meta attribute'
@@ -14,9 +18,6 @@ class Command(BaseCommand):
if not os.path.exists(output_dir):
os.makedirs(output_dir, exist_ok=True)
- base_uri = getattr(settings, 'BASE_URL', '')
- max_depth = getattr(settings, 'MAX_RECURSION_DEPTH', 5)
- request_timeout = getattr(settings, 'SSR_REQUEST_TIMEOUT', 10)
for model in apps.get_models():
if hasattr(model._meta, 'static_version'):
--
GitLab
From 1140f26ca1d9cb022757153057ddb88560386fdc Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Mon, 10 Jun 2024 11:55:13 +0200
Subject: [PATCH 10/17] force: refetch
---
.../commands/generate_static_content.py | 42 +++++++++----------
1 file changed, 21 insertions(+), 21 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index 7b553797..a668a542 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -80,28 +80,28 @@ class Command(BaseCommand):
if not os.path.exists(associated_file_dir):
os.makedirs(associated_file_dir)
- if not os.path.exists(associated_file_path):
+ # if not os.path.exists(associated_file_path):
# Fetch associated content
- try:
- response = requests.get(associated_url, timeout=request_timeout)
- if response.status_code == 200:
- associated_content = self.update_ids_and_fetch_associated(response.text, base_uri, output_dir, depth + 1, max_depth)
- print(f"associated_file_path: {associated_file_path}")
- associated_file_dir = os.path.dirname(associated_file_path)
- print(f"associated_file_path: {associated_file_dir}")
- if not os.path.exists(associated_file_dir):
- os.makedirs(associated_file_dir)
- with open(associated_file_path, 'w') as f:
- f.write(associated_content)
- self.stdout.write(self.style.SUCCESS(f'Successfully fetched and saved associated content for {associated_url}'))
- else:
- self.stdout.write(self.style.ERROR(f'Failed to fetch associated content from {associated_url}: {response.status_code}'))
- except requests.exceptions.Timeout:
- self.stdout.write(self.style.ERROR(f'Request to {associated_url} timed out'))
- except requests.exceptions.RequestException as e:
- self.stdout.write(self.style.ERROR(f'An error occurred: {e}'))
- else:
- self.stdout.write(self.style.SUCCESS(f'Associated file already exists for {associated_url}, skipping fetch'))
+ try:
+ response = requests.get(associated_url, timeout=request_timeout)
+ if response.status_code == 200:
+ associated_content = self.update_ids_and_fetch_associated(response.text, base_uri, output_dir, depth + 1, max_depth)
+ print(f"associated_file_path: {associated_file_path}")
+ associated_file_dir = os.path.dirname(associated_file_path)
+ print(f"associated_file_path: {associated_file_dir}")
+ if not os.path.exists(associated_file_dir):
+ os.makedirs(associated_file_dir)
+ with open(associated_file_path, 'w') as f:
+ f.write(associated_content)
+ self.stdout.write(self.style.SUCCESS(f'Successfully fetched and saved associated content for {associated_url}'))
+ else:
+ self.stdout.write(self.style.ERROR(f'Failed to fetch associated content from {associated_url}: {response.status_code}'))
+ except requests.exceptions.Timeout:
+ self.stdout.write(self.style.ERROR(f'Request to {associated_url} timed out'))
+ except requests.exceptions.RequestException as e:
+ self.stdout.write(self.style.ERROR(f'An error occurred: {e}'))
+ # else:
+ # self.stdout.write(self.style.SUCCESS(f'Associated file already exists for {associated_url}, skipping fetch'))
for key, value in item.items():
if isinstance(value, dict):
--
GitLab
From 47ac4f980f489e1eb402ff88794d8fe7e43ff285 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Mon, 10 Jun 2024 17:46:52 +0200
Subject: [PATCH 11/17] fix: cache-control
---
djangoldp/views.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/djangoldp/views.py b/djangoldp/views.py
index 867a658a..f88516df 100644
--- a/djangoldp/views.py
+++ b/djangoldp/views.py
@@ -630,6 +630,11 @@ def serve_static_content(request, path):
content = file.read()
json_content = json.loads(content)
- return JsonResponse(json_content, safe=False, status=200, content_type='application/ld+json', headers={'Access-Control-Allow-Origin': '*'})
+ return JsonResponse(json_content, safe=False, status=200,
+ content_type='application/ld+json',
+ headers={
+ 'Access-Control-Allow-Origin': '*',
+ 'Cache-Control': 'public, max-age=3600',
+ })
else:
return HttpResponseNotFound('File not found')
--
GitLab
From 388ab16c658b1dbadad49dc1aa675a69d3fd47c2 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 21 Jun 2024 15:18:44 +0200
Subject: [PATCH 12/17] cleaning: views and urls
---
.../management/commands/generate_static_content.py | 11 +++--------
djangoldp/urls.py | 1 -
djangoldp/views.py | 1 -
3 files changed, 3 insertions(+), 10 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index a668a542..3023a2db 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -62,7 +62,6 @@ class Command(BaseCommand):
else:
self.update_and_fetch_id(data, base_uri, output_dir, depth, max_depth)
- print(f"Content: {data}")
return json.dumps(data)
except json.JSONDecodeError as e:
self.stdout.write(self.style.ERROR(f'Failed to decode JSON: {e}'))
@@ -73,22 +72,20 @@ class Command(BaseCommand):
parsed_url = urlparse(item['@id'])
path = f'/ssr{parsed_url.path}'
item['@id'] = urlunparse((parsed_url.scheme, parsed_url.netloc, path, parsed_url.params, parsed_url.query, parsed_url.fragment))
+
associated_url = urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment))
associated_file_path = path[1:-1] + '.json'
- print(f"associated_file_path: {associated_file_path}")
associated_file_dir = os.path.dirname(associated_file_path)
+
if not os.path.exists(associated_file_dir):
os.makedirs(associated_file_dir)
- # if not os.path.exists(associated_file_path):
- # Fetch associated content
try:
response = requests.get(associated_url, timeout=request_timeout)
if response.status_code == 200:
associated_content = self.update_ids_and_fetch_associated(response.text, base_uri, output_dir, depth + 1, max_depth)
- print(f"associated_file_path: {associated_file_path}")
associated_file_dir = os.path.dirname(associated_file_path)
- print(f"associated_file_path: {associated_file_dir}")
+
if not os.path.exists(associated_file_dir):
os.makedirs(associated_file_dir)
with open(associated_file_path, 'w') as f:
@@ -100,8 +97,6 @@ class Command(BaseCommand):
self.stdout.write(self.style.ERROR(f'Request to {associated_url} timed out'))
except requests.exceptions.RequestException as e:
self.stdout.write(self.style.ERROR(f'An error occurred: {e}'))
- # else:
- # self.stdout.write(self.style.SUCCESS(f'Associated file already exists for {associated_url}, skipping fetch'))
for key, value in item.items():
if isinstance(value, dict):
diff --git a/djangoldp/urls.py b/djangoldp/urls.py
index 075dddec..defa166b 100644
--- a/djangoldp/urls.py
+++ b/djangoldp/urls.py
@@ -37,7 +37,6 @@ urlpatterns = [
permission_classes=[ReadOnly], )),
re_path(r'^\.well-known/webfinger/?$', WebFingerView.as_view()),
path('inbox/', InboxView.as_view()),
- # path('ssr/<str:filename>/', serve_static_content, name='serve_static_content'),
re_path(r'^ssr/(?P<path>.*)$', serve_static_content, name='serve_static_content'),
]
diff --git a/djangoldp/views.py b/djangoldp/views.py
index f88516df..c90eef1e 100644
--- a/djangoldp/views.py
+++ b/djangoldp/views.py
@@ -624,7 +624,6 @@ def serve_static_content(request, path):
if not file_path.endswith('.json'):
file_path += '.json'
- print(f"file_path: {file_path}")
if os.path.exists(file_path):
with open(file_path, 'r') as file:
content = file.read()
--
GitLab
From 1507e34593d6fcb2d198d4d5c31cd227c11a4d79 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 21 Jun 2024 16:09:39 +0200
Subject: [PATCH 13/17] adding: command documentation
---
docs/commands.md | 41 +++++++++++++++++++++++++++++++++++++++++
docs/create_model.md | 3 ++-
2 files changed, 43 insertions(+), 1 deletion(-)
create mode 100644 docs/commands.md
diff --git a/docs/commands.md b/docs/commands.md
new file mode 100644
index 00000000..357305d4
--- /dev/null
+++ b/docs/commands.md
@@ -0,0 +1,41 @@
+# Available commands
+
+##Â generate_static_content
+
+You can generate and make available at a /ssr/xxx URI a static copy of the AnonymousUser view of given models.
+Those models need to be configured with the `static_version` and `static_params` Meta options like:
+
+```python
+class Location(Model):
+ name = models.CharField(max_length=255)
+ address = models.CharField(max_length=255)
+ zip_code = models.IntegerField()
+ visible = models.BooleanField(default=False)
+
+ class Meta:
+ # Allow generating a static version of the container view
+ static_version = 1
+
+ # Add some GET parameters to configure the selection of data
+ static_params = {
+ "search-fields": "visible",
+ "search-terms": True,
+ "search-method": "exact"
+ }
+```
+
+You will need additional settings defined either in your settings.yml or settings.py file:
+
+```yml
+BASE_URL: 'http://localhost:8000/'
+MAX_RECURSION_DEPTH: 10 # Default value: 5
+SSR_REQUEST_TIME: 20 # Default value 10 (seconds)
+```
+
+Then you can try it out by executing the following command:
+
+```sh
+python manage.py generate_static_content
+```
+
+You can also set a cron task or a celery Task to launch this command in a regular basis.
\ No newline at end of file
diff --git a/docs/create_model.md b/docs/create_model.md
index 9297ba52..50fffc9f 100644
--- a/docs/create_model.md
+++ b/docs/create_model.md
@@ -1,5 +1,5 @@
-### User model requirements
+# User model requirements
When implementing authentication in your own application, you have two options:
@@ -469,3 +469,4 @@ MIDDLEWARE = [
```
Notice that it will return only HTTP 200 Code.
+
--
GitLab
From f973b5bdb7bc3df9bb821deac97d7d5d5ae0f2d7 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 21 Jun 2024 16:14:56 +0200
Subject: [PATCH 14/17] update: comments
---
.../management/commands/generate_static_content.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index 3023a2db..9e2fe36e 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -21,19 +21,19 @@ class Command(BaseCommand):
for model in apps.get_models():
if hasattr(model._meta, 'static_version'):
- print(f"model: {model}")
+ print(f"Generating content for model: {model}")
container_path = model.get_container_path()
url = f'{base_uri}{container_path}'
- print(f"current request url before adding params: {url}")
+ print(f"Current request url before adding params: {url}")
if hasattr(model._meta, 'static_params'):
- # static_params which is a json must be decomposed and added to the url as query parameters, first with ? then with &
+ # static_params are added to the url as query parameters
url += '?'
for key, value in model._meta.static_params.items():
url += f'{key}={value}&'
url = url[:-1]
- print(f"current request url after adding params: {url}")
+ print(f"Current request url after adding params: {url}")
response = requests.get(url, timeout=request_timeout)
if response.status_code == 200:
@@ -43,7 +43,7 @@ class Command(BaseCommand):
filename = container_path[1:-1]
file_path = os.path.join(output_dir, f'{filename}.json')
- print(f"file_path: {file_path}")
+ print(f"Output file_path: {file_path}")
with open(file_path, 'w') as f:
f.write(content)
self.stdout.write(self.style.SUCCESS(f'Successfully fetched and saved content for {model._meta.model_name} from {url}'))
--
GitLab
From fb0968a43afc682c6b5e963a521aca15bc151b53 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 21 Jun 2024 16:15:49 +0200
Subject: [PATCH 15/17] fix: comment
---
djangoldp/management/commands/generate_static_content.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index 9e2fe36e..cdb44479 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -11,7 +11,7 @@ max_depth = getattr(settings, 'MAX_RECURSION_DEPTH', 5)
request_timeout = getattr(settings, 'SSR_REQUEST_TIMEOUT', 10)
class Command(BaseCommand):
- help = 'Generate static content for models with a specific meta attribute'
+ help = 'Generate static content for models having the static_version meta attribute set to 1/true'
def handle(self, *args, **kwargs):
output_dir = 'ssr'
--
GitLab
From 21d0605c7e460f69eee3e6e6cf0a8d32ec9df0e5 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 21 Jun 2024 16:22:53 +0200
Subject: [PATCH 16/17] switch: to jsonld extension
---
djangoldp/management/commands/generate_static_content.py | 4 ++--
djangoldp/views.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/djangoldp/management/commands/generate_static_content.py b/djangoldp/management/commands/generate_static_content.py
index cdb44479..2cce19e3 100644
--- a/djangoldp/management/commands/generate_static_content.py
+++ b/djangoldp/management/commands/generate_static_content.py
@@ -41,7 +41,7 @@ class Command(BaseCommand):
content = self.update_ids_and_fetch_associated(content, base_uri, output_dir, 0, max_depth)
filename = container_path[1:-1]
- file_path = os.path.join(output_dir, f'{filename}.json')
+ file_path = os.path.join(output_dir, f'{filename}.jsonld')
print(f"Output file_path: {file_path}")
with open(file_path, 'w') as f:
@@ -74,7 +74,7 @@ class Command(BaseCommand):
item['@id'] = urlunparse((parsed_url.scheme, parsed_url.netloc, path, parsed_url.params, parsed_url.query, parsed_url.fragment))
associated_url = urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment))
- associated_file_path = path[1:-1] + '.json'
+ associated_file_path = path[1:-1] + '.jsonld'
associated_file_dir = os.path.dirname(associated_file_path)
if not os.path.exists(associated_file_dir):
diff --git a/djangoldp/views.py b/djangoldp/views.py
index c90eef1e..4a644752 100644
--- a/djangoldp/views.py
+++ b/djangoldp/views.py
@@ -621,8 +621,8 @@ class WebFingerView(View):
def serve_static_content(request, path):
file_path = os.path.join('ssr', path[:-1])
# Transform file_path to valid filename by adding .json extension
- if not file_path.endswith('.json'):
- file_path += '.json'
+ if not file_path.endswith('.jsonld'):
+ file_path += '.jsonld'
if os.path.exists(file_path):
with open(file_path, 'r') as file:
--
GitLab
From 8292e068288a550e1b111d334d3ed5983c45ab15 Mon Sep 17 00:00:00 2001
From: autodeploy <benoit@startinblox.com>
Date: Fri, 21 Jun 2024 16:39:30 +0200
Subject: [PATCH 17/17] fix: remove useless message
---
djangoldp/views.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/djangoldp/views.py b/djangoldp/views.py
index 4a644752..40897690 100644
--- a/djangoldp/views.py
+++ b/djangoldp/views.py
@@ -620,7 +620,6 @@ class WebFingerView(View):
def serve_static_content(request, path):
file_path = os.path.join('ssr', path[:-1])
- # Transform file_path to valid filename by adding .json extension
if not file_path.endswith('.jsonld'):
file_path += '.jsonld'
--
GitLab