.gitignore

 dist
+build
 *.egg-info
+*.eggs
 *.pyc
 *~
+*.swp
+djangoldp/tests/tests_temp.py
+*/.idea/*
+.DS_STORE
+venv

.gitlab-ci.yml

 ---
-image: python:3.6
+image: python:3.11
 
-stages:
-  - test
-  - release
+include:
+  project: infra/gitlab
+  ref: master
+  file: templates/python.ci.yml
 
 test:
   stage: test
@@ -11,22 +12,22 @@ test:
     - pip install .[dev]
     - python -m unittest djangoldp.tests.runner
   except:
-    - master
     - tags
   tags:
     - test
 
-publish:
-  stage: release
-  before_script:
-    - pip install python-semantic-release~=5.0 sib-commit-parser~=0.3
-    - git config user.name "${GITLAB_USER_NAME}"
-    - git config user.email "${GITLAB_USER_EMAIL}"
-    - git remote set-url origin "https://gitlab-ci-token:${GL_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
-    - git fetch --tags
+crypto-test:
+  stage: test
   script:
-    - semantic-release publish
-  only:
-    - master
+    - pip install .[crypto]
+    - python -m unittest djangoldp_crypto.tests.runner
+  except:
+    - tags
   tags:
-    - deploy
+    - test
+
+publish:
+  cache: []
+  extends: .publish_pypi
+
+

CHANGELOG.md

+# Changelog
+
+<!--next-version-placeholder-->
+
+## v3.0.5 (2023-07-24)
+
+### Fix
+
+* Readme ([`8792530`](https://github.com/djangoldp-packages/djangoldp/commit/87925305b2093282230511bceb38978db4b279a2))
+
+## v3.0.4 (2023-07-24)
+
+### Fix
+
+* Readme ([`eeedc33`](https://github.com/djangoldp-packages/djangoldp/commit/eeedc3378ed3f4e454377431da6bb50202efdcdc))
+
+## v3.0.3 (2023-07-24)
+
+### Fix
+
+* Readme ([`73597b6`](https://github.com/djangoldp-packages/djangoldp/commit/73597b65430a4d23306f78def0331bda60857493))
+
+
+## Unreleased
+
+* Imported CLI along with development template from sib-manager project

Dockerfile

+# use with your own settings.yml
+FROM python:3.11
+LABEL maintainer="Plup <plup@plup.io>"
+
+# get server
+RUN pip install djangoldp
+
+# create a server instance
+RUN djangoldp initserver ldpserver
+WORKDIR /ldpserver
+#COPY settings.yml .
+RUN djangoldp install
+RUN djangoldp configure --with-dummy-admin
+
+# run the server
+EXPOSE 8000
+CMD ["djangoldp", "runserver"]

LICENSE

 MIT License
 
-Copyright (c) 2018 Startin blox
+Copyright (c) 2018-2023 Startin blox
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

MANIFEST.in

+graft djangoldp/conf/package_template
+graft djangoldp/conf/server_template
+graft djangoldp/templates/

djangoldp/__init__.py

 from django.db.models import options
 
 __version__ = '0.0.0'
+
 options.DEFAULT_NAMES += (
-    'lookup_field', 'rdf_type', 'rdf_context', 'auto_author', 'auto_author_field', 'owner_field', 'view_set',
-    'container_path', 'permission_classes', 'serializer_fields', 'serializer_fields_exclude', 'nested_fields',
-    'nested_fields_exclude', 'depth', 'anonymous_perms', 'authenticated_perms', 'owner_perms')
-default_app_config = 'djangoldp.apps.DjangoldpConfig'
+    'lookup_field', 'rdf_type', 'rdf_context', 'auto_author', 'owner_field', 'owner_urlid_field',
+    'view_set', 'container_path', 'permission_classes', 'serializer_fields', 'serializer_fields_exclude', 'empty_containers',
+    'nested_fields', 'depth', 'permission_roles', 'inherit_permissions', 'public_field', 'static_version', 'static_params', 'active_field', 'disable_url')

djangoldp/activities/services.py

 import threading
 import json
 import time
+import copy
 import requests
 from queue import Queue
 from requests.exceptions import Timeout, ConnectionError
-from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util.retry import Retry
 from urllib.parse import urlparse
 from django.contrib.auth import get_user_model
 from django.db.models.signals import post_save, post_delete, m2m_changed
+from django.db.models import Q
 from django.dispatch import receiver, Signal
 from django.conf import settings
 from rest_framework.utils import model_meta
@@ -31,13 +31,92 @@ SCHEDULER_SETTINGS = {
 
 MAX_ACTIVITY_RESCHEDULES = getattr(settings, 'MAX_ACTIVITY_RESCHEDULES', 3)
 DEFAULT_BACKOFF_FACTOR = getattr(settings, 'DEFAULT_BACKOFF_FACTOR', 1)
-DEFAULT_ACTIVITY_DELAY = getattr(settings, 'DEFAULT_ACTIVITY_DELAY', 3)
+DEFAULT_ACTIVITY_DELAY = getattr(settings, 'DEFAULT_ACTIVITY_DELAY', 0.1)
 DEFAULT_REQUEST_TIMEOUT = getattr(settings, 'DEFAULT_REQUEST_TIMEOUT', 10)
+MAX_RECORDS_ACTIVITY_CACHE = getattr(settings, 'MAX_RECORDS_ACTIVITY_CACHE', 10000)
 
 
 activity_sending_finished = Signal()
 
 
+class ActivityInMemoryCache:
+    '''
+    {
+        'urlid': {
+            # if an object is sent without a urlid it is not cached
+            'object_urlid': {
+                # for create/update, just interested in the most recent activity on this external id
+                './': ACTIVITY
+
+                # for add/remove, we're interested also in the target (container_id)
+                'circles': ACTIVITY
+                'circle-members': ACTIVITY
+            }
+        }
+    }
+    '''
+
+    def __init__(self):
+        self.cache = {
+        }
+
+    def reset(self):
+        self.cache = {
+        }
+
+    def has(self, urlid, object_id, target_id=None):
+        if urlid not in self.cache or object_id not in self.cache[urlid]:
+            return False
+    
+        if target_id is None:
+            target_id = './'
+        
+        return target_id in self.cache[urlid][object_id]
+
+    def get(self, urlid, object_id, target_id=None):
+        if target_id is None:
+            target_id = './'
+
+        if self.has(urlid, object_id, target_id):
+            return self.cache[urlid][object_id][target_id]
+        else:
+            return None
+
+    def set(self, urlid, object_id, target_id=None, value=None):
+        if MAX_RECORDS_ACTIVITY_CACHE == 0:
+            return
+
+        if len(self.cache.keys()) > MAX_RECORDS_ACTIVITY_CACHE:
+            self.reset()
+
+        if target_id is None:
+            target_id = './'
+
+        if urlid not in self.cache:
+            self.cache[urlid] = {}
+        
+        if object_id not in self.cache[urlid]:
+            self.cache[urlid][object_id] = {}
+        
+        self.cache[urlid][object_id][target_id] = copy.deepcopy(value)
+
+    def invalidate(self, urlid, object_id=None, target_id=None):
+        # can clear the cache for an entire record or at any level in the cache
+        if object_id is not None:
+            if target_id is not None:
+                self.cache[urlid][object_id].pop(target_id, None)
+            else:
+                self.cache[urlid].pop(object_id, None)
+        else:
+            self.cache.pop(urlid, None)
+
+
+# used to minimise the activity traffic to necessary activities,
+# preferred over a database solution which is slower
+ACTIVITY_CACHE = ActivityInMemoryCache()
+ACTIVITY_SAVING_SETTING = getattr(settings, 'STORE_ACTIVITIES', 'ERROR')
+
+
 class ActivityQueueService:
     '''Manages an asynchronous queue for Activity format messages'''
     initialized = False
@@ -96,6 +175,13 @@ class ActivityQueueService:
             return {'data': {}}
         return requests.post(url, data=json.dumps(activity), headers=headers, timeout=timeout)
 
+    @classmethod
+    def _get_str_urlid(cls, obj):
+        if obj is None:
+            return None
+
+        return obj if isinstance(obj, str) else obj['@id'] if '@id' in obj else None
+
     @classmethod
     def _save_activity_from_response(cls, response, url, scheduled_activity):
         '''
@@ -107,13 +193,27 @@ class ActivityQueueService:
         if hasattr(response, 'text'):
             response_body = response.text
         response_location = getattr(response, "Location", None)
-        status_code = getattr(response, "status_code", None)
-        success = str(status_code).startswith("2")
-
-        return cls._save_sent_activity(scheduled_activity.to_activitystream(), ActivityModel, success=success,
-                                       external_id=url, type=scheduled_activity.type,
-                                       response_location=response_location, response_code=str(status_code),
-                                       response_body=response_body)
+        status_code = getattr(response, "status_code", response['status_code'] if 'status_code' in response else None)
+        success = str(status_code).startswith("2") if status_code is not None else False
+        
+        # strip some key info from the activity (standardise the datatype)
+        activity_json = scheduled_activity.to_activitystream() if isinstance(scheduled_activity, ScheduledActivity) else scheduled_activity
+        activity_type = scheduled_activity.type if hasattr(scheduled_activity, 'type') else scheduled_activity['type']
+
+        # set the activity cache
+        if 'object' in activity_json:
+            object_id = cls._get_str_urlid(activity_json['object'])
+            
+            if object_id is not None:
+                target_origin = cls._get_str_urlid(activity_json.get('target', activity_json.get('origin', None)))
+                ACTIVITY_CACHE.set(url, object_id, target_origin, activity_json)
+
+        # save the activity if instructed to do so
+        if ACTIVITY_SAVING_SETTING == 'VERBOSE' or (not success and ACTIVITY_SAVING_SETTING == 'ERROR'):
+            return cls._save_sent_activity(activity_json, ActivityModel, success=success,
+                                           external_id=url, type=activity_type,
+                                           response_location=response_location, response_code=str(status_code),
+                                           response_body=response_body)
 
     @classmethod
     def _attempt_failed_reschedule(cls, url, scheduled_activity, backoff_factor):
@@ -179,7 +279,7 @@ class ActivityQueueService:
             if type is None:
                 return []
             type = type.lower()
-            group_a = ['create', 'update', 'delete']
+            group_a = ['create', 'update', 'delete', 'creation', 'deletion']
             groub_b = ['add', 'remove']
 
             if type in group_a:
@@ -241,22 +341,17 @@ class ActivityQueueService:
             '''returns False if the two activities are equivalent'''
             return ordered(old_activity['object']) == ordered(new_activity['object'])
 
-        def get_most_recent_sent_activity(external_id):
-            '''returns the most recently sent activity which meets the specification'''
-            activities = ActivityModel.objects.filter(external_id=external_id, is_finished=True,
-                                                      type__in=['create', 'update']).order_by('-created_at')[:10]
-            for a in activities.all():
-                a = a.to_activitystream()
-                if 'object' in a:
-                    return a
-            return None
+        def get_most_recent_sent_activity(external_id, object_id):
+            return ACTIVITY_CACHE.get(external_id, object_id)
 
         # str objects will have to be checked manually by the receiver
         new_activity = scheduled_activity.to_activitystream()
-        if 'object' not in new_activity or isinstance(new_activity['object'], str):
+        if 'object' not in new_activity or isinstance(new_activity['object'], str) or \
+             '@id' not in new_activity['object']:
+        
             return True
 
-        old_activity = get_most_recent_sent_activity(url)
+        old_activity = get_most_recent_sent_activity(url, new_activity['object']['@id'])
 
         if old_activity is None:
             return True
@@ -269,33 +364,25 @@ class ActivityQueueService:
     def _add_remove_is_new(cls, url, scheduled_activity):
         '''auxiliary function validates if the receiver does not know about this Add/Remove activity'''
         def get_most_recent_sent_activity(source_obj, source_target_origin):
-            # get a list of activities with the right type
-            activities = ActivityModel.objects.filter(external_id=url, is_finished=True,
-                                                      type__in=['add', 'remove']).order_by('-created_at')[:10]
-
-            # we are searching for the most recent Add/Remove activity which shares inbox, object and target/origin
-            for a in activities.all():
-                astream = a.to_activitystream()
-                obj = astream.get('object', None)
-                target_origin = astream.get('target', astream.get('origin', None))
-                if obj is None or target_origin is None:
-                    continue
+            obj_id = cls._get_str_urlid(source_obj)
+            target_id = cls._get_str_urlid(source_target_origin)
 
-                if source_obj == obj and source_target_origin == target_origin:
-                    return a
-            return None
+            return ACTIVITY_CACHE.get(url, obj_id, target_id)
 
         new_activity = scheduled_activity.to_activitystream()
         new_obj = new_activity.get('object', None)
         new_target_origin = new_activity.get('target', new_activity.get('origin', None))
 
         # bounds checking
-        if new_obj is None or new_target_origin is None:
+        if new_obj is None or new_target_origin is None or \
+            (not isinstance(new_obj, str) and '@id' not in new_obj) or \
+            (not isinstance(new_target_origin, str) and '@id' not in new_target_origin):
             return True
 
         #  if most recent is the same type of activity as me, it's not new
         old_activity = get_most_recent_sent_activity(new_obj, new_target_origin)
-        if old_activity is not None and old_activity.type == scheduled_activity.type:
+
+        if old_activity is not None and old_activity['type'].lower() == scheduled_activity.type.lower():
             return False
         return True
 
@@ -342,11 +429,11 @@ class ActivityQueueService:
                             response_location=None, response_code=None, local_id=None, response_body=None):
         '''
         Auxiliary function saves a record of parameterised activity
-        :param model_represenation: the model class which should be used to store the activity. Defaults to djangoldp.Activity, must be a subclass
+        :param model_represenation: the model class which should be used to store the activity. Defaults to djangol.Activity, must be a subclass
         '''
-        payload = bytes(json.dumps(activity), "utf-8")
+        payload = json.dumps(activity)
         if response_body is not None:
-            response_body = bytes(json.dumps(response_body), "utf-8")
+            response_body = json.dumps(response_body)
         if local_id is None:
             local_id = settings.SITE_URL + "/outbox/"
         if type is not None:
@@ -372,7 +459,7 @@ class ActivityPubService(object):
             return
 
         obj = {
-            "@type": Model.get_model_rdf_type(model),
+            "@type": getattr(model._meta, "rdf_type", None),
             "@id": instance.urlid
         }
         if obj['@type'] is None:
@@ -384,10 +471,13 @@ class ActivityPubService(object):
                 value = getattr(instance, field_name, None)
                 if value is None:
                     continue
-
+                
+                if not hasattr(value, 'urlid'):
+                    continue
+                
                 sub_object = {
                     "@id": value.urlid,
-                    "@type": Model.get_model_rdf_type(type(value))
+                    "@type": getattr(value._meta, "rdf_type", None)
                 }
 
                 if sub_object['@type'] is None:
@@ -506,7 +596,7 @@ class ActivityPubService(object):
         info = model_meta.get_field_info(sender)
 
         # bounds checking
-        if not hasattr(instance, 'urlid') or Model.get_model_rdf_type(sender) is None:
+        if not hasattr(instance, 'urlid') or getattr(sender._meta, "rdf_type", None) is None:
             return set()
 
         # check each foreign key for a distant resource
@@ -515,7 +605,7 @@ class ActivityPubService(object):
             if not relation_info.to_many:
                 value = getattr(instance, field_name, None)
                 if value is not None and Model.is_external(value):
-                    target_type = Model.get_model_rdf_type(type(value))
+                    target_type = getattr(value._meta, "rdf_type", None)
 
                     if target_type is None:
                         continue
@@ -533,9 +623,9 @@ class ActivityPubService(object):
         return inboxes
 
     @classmethod
-    def get_follower_inboxes(cls, object_urlid):
+    def get_follower_inboxes(cls, object_urlid, object_container=None):
         '''Auxiliary function returns a set of inboxes, from the followers of parameterised object urlid'''
-        inboxes = set(Follower.objects.filter(object=object_urlid).values_list('inbox', flat=True))
+        inboxes = set(Follower.objects.filter(Q(object=object_urlid) | Q(object__endswith=object_container)).values_list('inbox', flat=True))
         return inboxes
 
     @classmethod
@@ -575,7 +665,7 @@ def check_save_for_backlinks(sender, instance, created, **kwargs):
             and not Model.is_external(instance) \
             and getattr(instance, 'username', None) != 'hubl-workaround-493':
         external_urlids = ActivityPubService.get_related_externals(sender, instance)
-        inboxes = ActivityPubService.get_follower_inboxes(instance.urlid)
+        inboxes = ActivityPubService.get_follower_inboxes(instance.urlid, instance.get_container_path())
         targets = set().union(ActivityPubService.get_target_inboxes(external_urlids), inboxes)
 
         if len(targets) > 0:
@@ -598,13 +688,13 @@ def check_save_for_backlinks(sender, instance, created, **kwargs):
 def check_delete_for_backlinks(sender, instance, **kwargs):
     if getattr(settings, 'SEND_BACKLINKS', True) and getattr(instance, 'allow_create_backlink', False) \
             and getattr(instance, 'username', None) != 'hubl-workaround-493':
-        targets = ActivityPubService.get_follower_inboxes(instance.urlid)
+        targets = ActivityPubService.get_follower_inboxes(instance.urlid, instance.get_container_path())
 
         if len(targets) > 0:
             for target in targets:
                 ActivityPubService.send_delete_activity(BACKLINKS_ACTOR, {
                     "@id": instance.urlid,
-                    "@type": Model.get_model_rdf_type(sender)
+                    "@type": getattr(instance._meta, "rdf_type", None)
                 }, target)
 
     # remove any Followers on this resource
@@ -640,8 +730,8 @@ def check_m2m_for_backlinks(sender, instance, action, *args, **kwargs):
         # we can only send backlinks on pre_clear because on post_clear the objects are gone
         if action != "pre_clear" and pk_set is None:
             return
-        member_rdf_type = Model.get_model_rdf_type(member_model)
-        container_rdf_type = Model.get_model_rdf_type(type(instance))
+        member_rdf_type = getattr(member_model._meta, "rdf_type", None)
+        container_rdf_type = getattr(instance._meta, "rdf_type", None)
 
         if member_rdf_type is None:
             return
@@ -650,7 +740,12 @@ def check_m2m_for_backlinks(sender, instance, action, *args, **kwargs):
 
         # build list of targets (models affected by the change)
         if action == "pre_clear":
-            pk_set = sender.objects.all().values_list(member_model.__name__.lower(), flat=True)
+            sender_info = model_meta.get_field_info(sender)
+            for field_name, relation_info in sender_info.relations.items():
+                if relation_info.related_model == member_model:
+                    pk_set = sender.objects.all().values_list(field_name, flat=True)
+        if pk_set is None or len(pk_set) == 0:
+            return
         query_set = member_model.objects.filter(pk__in=pk_set)
         targets = build_targets(query_set)
 

djangoldp/activities/verbs.py

@@ -33,12 +33,24 @@ class Activity(Object):
             delattr(new, "bcc")
         return new
 
+    def _validate_type_id_defined(self, value):
+        '''recursively ensures that all nested dict items define @id and @type attributes'''
+        for item in value.items():
+            if isinstance(item[1], dict):
+                item_value = item[1]
+                if '@type' not in item_value or '@id' not in item_value:
+                    raise errors.ActivityStreamValidationError("all sub-objects passed in activity object must define @id and @type tags")
+                self._validate_type_id_defined(item_value)
+
+
     def validate(self):
         for attr in self.required_attributes.keys():
             if not isinstance(getattr(self, attr, None), self.required_attributes[attr]):
                 raise errors.ActivityStreamValidationError("required attribute " + attr + " of type "
                                                            + str(self.required_attributes[attr]))
 
+        # validate that every dictionary stored in object has @id and @type
+        self._validate_type_id_defined(self.__getattribute__("object"))
 
 class Add(Activity):
     type = "Add"

djangoldp/admin.py

+from csv import DictWriter
 from django.contrib import admin
-from guardian.admin import GuardedModelAdmin
 from django.contrib.auth.admin import UserAdmin
-from djangoldp.models import Activity, ScheduledActivity
+from django.core.exceptions import FieldDoesNotExist
+from django.http import HttpResponse
+from guardian.admin import GuardedModelAdmin
+from djangoldp.models import Activity, ScheduledActivity, Follower
+from djangoldp.activities.services import ActivityQueueService
 
 
 class DjangoLDPAdmin(GuardedModelAdmin):
@@ -9,13 +13,38 @@ class DjangoLDPAdmin(GuardedModelAdmin):
     An admin model representing a federated object. Inherits from GuardedModelAdmin to provide Django-Guardian
     object-level permissions
     '''
-    pass
-
-
-class DjangoLDPUserAdmin(UserAdmin, GuardedModelAdmin):
+    actions = ['export_csv']
+    export_fields = []
+
+    def resolve_verbose_name(self, field_path):
+        field = self
+        for field_name in field_path.split('__'):
+            try:
+                field = field.model._meta.get_field(field_name)
+            except FieldDoesNotExist:
+                return None
+        return field.verbose_name
+
+    @admin.action(description="Export CSV")
+    def export_csv(self, request, queryset):
+        response = HttpResponse(content_type="text/csv")
+        response['Content-Disposition'] = f'attachment; filename="{self.model.__name__}.csv"'
+        # only keep fields that can be resolved, keep only urlid if none
+        field_list = list(filter(self.resolve_verbose_name, self.export_fields or self.list_display)) or ['urlid']
+        headers = {field:self.resolve_verbose_name(field) for field in field_list}
+
+        writer = DictWriter(response, fieldnames=field_list)
+        writer.writerow(headers)
+        writer.writerows(queryset.values(*field_list))
+        return response
+
+
+class DjangoLDPUserAdmin(UserAdmin, DjangoLDPAdmin):
     '''An extension of UserAdmin providing the functionality of DjangoLDPAdmin'''
 
     list_display = ('urlid', 'email', 'first_name', 'last_name', 'date_joined', 'last_login', 'is_staff')
+    search_fields = ['urlid', 'email', 'first_name', 'last_name']
+    ordering = ['urlid']
 
     def get_fieldsets(self, request, obj=None):
         fieldsets = super().get_fieldsets(request, obj)
@@ -35,11 +64,21 @@ class DjangoLDPUserAdmin(UserAdmin, GuardedModelAdmin):
         return fieldsets
 
 
+@admin.action(description='Resend activity')
+def resend_activity(modeladmin, request, queryset):
+    for a in queryset:
+        ActivityQueueService.send_activity(a.external_id, a.to_activitystream())
+resend_activity.short_description = 'Resend activity'
+
+
+@admin.register(Activity, ScheduledActivity)
 class ActivityAdmin(DjangoLDPAdmin):
     fields = ['urlid', 'type', 'local_id', 'external_id', 'created_at', 'success', 'payload_view', 'response_code',
               'response_location', 'response_body_view']
     list_display = ['created_at', 'type', 'local_id', 'external_id', 'success', 'response_code']
     readonly_fields = ['created_at', 'payload_view', 'response_location', 'response_code', 'response_body_view']
+    search_fields = ['urlid', 'type', 'local_id', 'external_id', 'response_code']
+    actions = [resend_activity]
 
     def payload_view(self, obj):
         return str(obj.to_activitystream())
@@ -48,5 +87,10 @@ class ActivityAdmin(DjangoLDPAdmin):
         return str(obj.response_to_json())
 
 
-admin.site.register(Activity, ActivityAdmin)
-admin.site.register(ScheduledActivity, ActivityAdmin)
+@admin.register(Follower)
+class FollowerAdmin(DjangoLDPAdmin):
+    fields = ['urlid', 'object', 'inbox', 'follower']
+    list_display = ['urlid', 'object', 'inbox', 'follower']
+    search_fields = ['object', 'inbox', 'follower']
+
+

djangoldp/apps.py

@@ -7,6 +7,18 @@ class DjangoldpConfig(AppConfig):
     def ready(self):
         self.auto_register_model_admin()
         self.start_activity_queue()
+        
+        # Patch guardian core to avoid prefetching permissions several times
+        from guardian.core import ObjectPermissionChecker
+        ObjectPermissionChecker._prefetch_cache_orig = ObjectPermissionChecker._prefetch_cache
+
+        def _prefetch_cache(self):
+            if hasattr(self.user, "_guardian_perms_cache"):
+                self._obj_perms_cache = self.user._guardian_perms_cache
+                return
+            self._prefetch_cache_orig()
+
+        ObjectPermissionChecker._prefetch_cache = _prefetch_cache
 
     def start_activity_queue(self):
         from djangoldp.activities.services import ActivityQueueService
@@ -22,6 +34,7 @@ class DjangoldpConfig(AppConfig):
         from django.conf import settings
         from django.contrib import admin
         from djangoldp.admin import DjangoLDPAdmin
+        from djangoldp.urls import get_all_non_abstract_subclasses
         from djangoldp.models import Model
 
         for package in settings.DJANGOLDP_PACKAGES:
@@ -36,9 +49,6 @@ class DjangoldpConfig(AppConfig):
             except ModuleNotFoundError:
                 pass
 
-        model_classes = {cls.__name__: cls for cls in Model.__subclasses__()}
-
-        for class_name in model_classes:
-            model_class = model_classes[class_name]
-            if not admin.site.is_registered(model_class):
-                admin.site.register(model_class, DjangoLDPAdmin)
+        for model in get_all_non_abstract_subclasses(Model):
+            if not admin.site.is_registered(model):
+                admin.site.register(model, DjangoLDPAdmin)

djangoldp/check_integrity.py

+'''
+  DjangoLDP Check Integrity
+  Usage `./manage.py check_integrity --help`
+'''
+from django.apps import apps
+from django.conf import settings
+from djangoldp.models import LDPSource
+from urllib.parse import urlparse
+import requests
+
+# Helper command for argument type checking
+def is_string(target):
+  if isinstance(target, str):
+    return target
+  return False
+
+# Helper command to check status code of a target
+def is_alive(target, status_code = 200):
+  return requests.get(target).status_code == status_code
+
+# Add argument to the `check_integrity` command
+def add_arguments(parser):
+  parser.add_argument(
+    "--ignore",
+    action="store",
+    default=False,
+    type=is_string,
+    help="Ignore any server, comma separated",
+  )
+  parser.add_argument(
+    "--ignore-faulted",
+    default=False,
+    nargs="?",
+    const=True,
+    help="Ignore eventual faulted",
+  )
+  parser.add_argument(
+    "--ignore-404",
+    default=False,
+    nargs="?",
+    const=True,
+    help="Ignore eventual 404",
+  )
+  parser.add_argument(
+    "--fix-faulted-resources",
+    default=False,
+    nargs="?",
+    const=True,
+    help="Fix faulted resources",
+  )
+  parser.add_argument(
+    "--fix-404-resources",
+    default=False,
+    nargs="?",
+    const=True,
+    help="Fix 404 resources",
+  )
+  parser.add_argument(
+    "--fix-offline-servers",
+    default=False,
+    nargs="?",
+    const=True,
+    help="Remove resources from offline servers",
+  )
+
+# Define our own checks
+def check_integrity(options):
+
+  models = apps.get_models()
+
+  ignored = set()
+  if(options["ignore"]):
+    for target in options["ignore"].split(","):
+      ignored.add(urlparse(target).netloc)
+
+  if(len(ignored) > 0):
+    print("Ignoring servers:")
+    for server in ignored:
+      print("- "+server)
+
+  resources = set()
+  resources_map = dict()
+  base_urls = set()
+
+  for model in models:
+    for obj in model.objects.all():
+      if hasattr(obj, "urlid"):
+        if(obj.urlid):
+          if(not obj.urlid.startswith(settings.BASE_URL)):
+            url = urlparse(obj.urlid).netloc
+            if(url not in ignored):
+              resources.add(obj.urlid)
+              resources_map[obj.urlid] = obj
+              base_urls.add(url)
+
+  if(len(base_urls) > 0):
+    print("Found "+str(len(resources_map))+" distant resources on "+str(len(models))+" models")
+    print("Servers that I have backlinks to:")
+    for server in base_urls:
+      print("- "+server)
+  else:
+    print("I don't have any backlink")
+
+  source_urls = set()
+  for source in LDPSource.objects.all():
+    source_urls.add(urlparse(source.urlid).netloc)
+
+  if(len(source_urls) > 0):
+    print("Servers that I'm allowed to get federated to:")
+    for server in source_urls:
+      print("- "+server)
+  else:
+    print("I'm not federated")
+
+  difference_servers = base_urls.difference(source_urls)
+  if(len(difference_servers) > 0):
+    print("Servers that I should not get aware of:")
+    for server in difference_servers:
+      print("- "+server)
+
+    # Handle faulted resources
+    if(not options["ignore_faulted"]):
+      faulted_resources = set()
+      for server in difference_servers:
+        for resource in resources:
+          if(urlparse(resource).netloc in server):
+            faulted_resources.add(resource)
+
+      if(len(faulted_resources) > 0):
+        print("Resources in fault:")
+        for resource in faulted_resources:
+          print("- "+resource)
+      else:
+        print("No resource are in fault")
+
+      if(options["fix_faulted_resources"]):
+        for resource in faulted_resources:
+          try:
+            resources_map[resource].delete()
+          except:
+            pass
+        print("Fixed faulted resources")
+      else:
+        print("Fix them with `./manage.py check_integrity --fix-faulted-resources`")
+  else:
+    print("I accept datas for every of those servers")
+
+  # Handle 404 resources
+  if(not options["ignore_404"]):
+    resources_404 = set()
+    resources_servers_offline = set()
+    for resource in resources:
+      try:
+        if(is_alive(resource, 404)):
+          resources_404.add(resource)
+      except:
+        resources_servers_offline.add(resource)
+
+    if(len(resources_404) > 0):
+      print("Faulted resources, 404:")
+      for resource in resources_404:
+        print("- "+resource)
+      if(options["fix_404_resources"]):
+        for resource in resources_404:
+          try:
+            resources_map[resource].delete()
+          except:
+            pass
+        print("Fixed 404 resources")
+      else:
+        print("Fix them with `./manage.py check_integrity --fix-404-resources`")
+
+    if(len(resources_servers_offline) > 0):
+      print("Faulted resources, servers offline:")
+      for resource in resources_servers_offline:
+        print("- "+resource)
+      if(options["fix_offline_servers"]):
+        for resource in resources_servers_offline:
+          try:
+            resources_map[resource].delete()
+          except:
+            pass
+        print("Fixed resources on offline servers")
+      else:
+        print("Fix them with `./manage.py check_integrity --fix-offline-servers`")
+
+    else:
+      print("No 404 in known resources")
\ No newline at end of file

djangoldp/cli.py

+import click
+import sys
+import yaml
+import subprocess
+from pkg_resources import resource_filename
+from pathlib import Path
+from django.core import management
+from django.core.management.base import CommandError
+from . import __version__
+
+
+# click entrypoint
+@click.group()
+@click.version_option(__version__)
+def main():
+
+    """DjangoLDP CLI"""
+
+
+@main.command()
+@click.argument('name', nargs=1, required=False)
+def initserver(name):
+
+    """Start a DjangoLDP server."""
+
+    try:
+        # use directly pwd
+        directory = Path.cwd()
+
+        if name:
+            # create a directory from project name in pwd
+            directory = Path.cwd() / name
+            directory.mkdir(parents=False, exist_ok=False)
+
+        # get the template path
+        template = resource_filename(__name__, 'conf/server_template')
+
+        # wrap the default django-admin startproject command
+        # this call import django settings and configure it
+        # see: https://docs.djangoproject.com/fr/2.2/topics/settings/#calling-django-setup-is-required-for-standalone-django-usage
+        # see: https://github.com/django/django/blob/stable/2.2.x/django/core/management/templates.py#L108
+        # fix: in 2.2 gabarit files options has been renamed: https://github.com/django/django/blob/stable/2.2.x/django/core/management/templates.py#L53
+        management.call_command('startproject', name, directory, template=template, files=['settings.yml'])
+
+    except FileExistsError:
+        click.echo(f'Error: the folder {directory} already exists')
+        sys.exit(1)
+
+    except CommandError as e:
+        click.echo(f'Error: {e}')
+        directory.rmdir()
+        sys.exit(1)
+
+@main.command()
+@click.argument('name', nargs=1)
+def startpackage(name):
+
+    """Start a DjangoLDP package."""
+
+    try:
+        # set directory
+        directory = Path.cwd() / name
+
+        # get the template path
+        template = resource_filename(__name__, 'conf/package_template')
+
+        # create dir
+        directory.mkdir(parents=False, exist_ok=False)
+
+        # wrap the default startapp command
+        management.call_command('startapp', name, directory, template=template)
+
+    except FileExistsError:
+        click.echo(f'Error: the folder {directory} already exists')
+        sys.exit(1)
+
+    except CommandError as e:
+        click.echo(f'Error: {e}')
+        sys.exit(1)
+
+@main.command()
+def install():
+
+    """Install project dependencies."""
+
+    try:
+        # load dependencies from config file
+        path = Path.cwd() / 'settings.yml'
+        with open(path, 'r') as f:
+            dependencies = yaml.safe_load(f).get('dependencies', [])
+
+        # install them by calling pip command
+        cmd = [sys.executable, "-m", "pip", "install", "--upgrade"]
+        try:
+            cmd.extend(dependencies)
+            subprocess.run(cmd).check_returncode()
+            click.echo('Installation done!')
+        except TypeError:
+            click.echo('No dependency to install')
+
+    except FileNotFoundError:
+        click.echo('Config error: no settings.yml file in this directory')
+        sys.exit(1)
+
+    except subprocess.CalledProcessError as e:
+        click.echo(f'Installation error: {e}')
+        sys.exit(1)
+
+
+@main.command()
+@click.option('--with-admin', 'admin', help='Create an administrator user with email.')
+@click.option('--email', help='Provide an email for administrator.')
+@click.option('--with-dummy-admin', 'dummy_admin', is_flag=True, help='Create a default "admin" user.')
+def configure(admin, dummy_admin, email):
+
+    """Configure the project."""
+
+    try:
+        # shortcut to the djangoldp.management command
+        path = str(Path.cwd() / 'manage.py')
+        cmd = [sys.executable, path, 'configure']
+        if admin:
+            if not email:
+                click.echo('Error: missing email for admin user')
+                return
+            cmd.extend(['--with-admin', admin, '--email', email])
+        elif dummy_admin:
+            cmd.append('--with-dummy-admin')
+        subprocess.run(cmd).check_returncode()
+
+        click.echo('Configuration done!')
+
+    except subprocess.CalledProcessError as e:
+        click.echo(f'Configuration error: {e}')
+        sys.exit(1)
+
+
+@main.command()
+def runserver():
+
+    """Run the Django embeded webserver."""
+
+    try:
+        # shortcut to the djangoldp.management command
+        path = str(Path.cwd() / 'manage.py')
+        cmd = [sys.executable, path, 'runserver', '0.0.0.0:8000']
+        subprocess.run(cmd).check_returncode()
+
+    except subprocess.CalledProcessError as e:
+        click.echo(f'Execution error: {e}')
+        sys.exit(1)

djangoldp/conf/__init__.py

+

djangoldp/conf/default_settings.py

+"""This module override some of default django configuration from the global settings."""
+
+from django.conf.global_settings import *
+
+####################
+# LDP              #
+####################
+
+LDP_RDF_CONTEXT = 'https://cdn.startinblox.com/owl/context.jsonld'
+
+MAX_ACTIVITY_RESCHEDULES = 3
+DEFAULT_BACKOFF_FACTOR = 1
+DEFAULT_ACTIVITY_DELAY = 0.2
+DEFAULT_REQUEST_TIMEOUT = 10
+
+LDP_INCLUDE_INNER_PERMS = False
+
+####################
+# CORE             #
+####################
+
+# https://en.wikipedia.org/wiki/List_of_tz_zones_by_name (although not all
+# systems may support all possibilities). When USE_TZ is True, this is
+# interpreted as the default user time zone.
+TIME_ZONE = 'UTC'
+
+# If you set this to True, Django will use timezone-aware datetimes.
+USE_TZ = True
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+# Database connection info. If left empty, will default to the dummy backend.
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': 'db.sqlite3',
+    }
+}
+
+# List of strings representing installed apps.
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'djangoldp',
+    'guardian'
+]
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+# Default server url
+SITE_URL = 'http://localhost:8000'
+BASE_URL = SITE_URL
+
+# Default URL conf
+ROOT_URLCONF = 'djangoldp.urls'
+
+# Absolute filesystem path to the directory that will hold user-uploaded files.
+# Example: "/var/www/example.com/media/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT.
+# Examples: "http://example.com/media/", "http://media.example.com/"
+MEDIA_URL = '/media/'
+
+# Absolute path to the directory static files should be collected to.
+# Example: "/var/www/example.com/static/"
+STATIC_ROOT = None
+
+# URL that handles the static files served from STATIC_ROOT.
+# Example: "http://example.com/static/", "http://static.example.com/"
+STATIC_URL = '/static/'
+
+# ETAGS config
+USE_ETAGS = True
+
+# Default X-Frame-Options header value
+X_FRAME_OPTIONS = 'DENY'
+
+# The Python dotted path to the WSGI application that Django's internal server
+# (runserver) will use. If `None`, the return value of
+# 'django.core.wsgi.get_wsgi_application' is used, thus preserving the same
+# behavior as previous versions of Django. Otherwise this should point to an
+# actual WSGI application object.
+WSGI_APPLICATION = 'server.wsgi.application'
+
+##############
+# MIDDLEWARE #
+##############
+
+# List of middleware to use. Order is important; in the request phase, these
+# middleware will be applied in the order given, and in the response
+# phase the middleware will be applied in reverse order.
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'djangoldp.middleware.AllowRequestedCORSMiddleware',
+    'django.middleware.gzip.GZipMiddleware',
+    'django_brotli.middleware.BrotliMiddleware'
+]
+
+##################
+# REST FRAMEWORK #
+##################
+REST_FRAMEWORK = {
+    'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema',
+    'DEFAULT_PAGINATION_CLASS': 'djangoldp.pagination.LDPPagination'
+}
+
+###################
+# DRF SPECTACULAR #
+###################
+ENABLE_SWAGGER_DOCUMENTATION = False
+SPECTACULAR_SETTINGS = {
+    'TITLE': 'DjangoLDP Based API Description',
+    'DESCRIPTION': 'Here you will find the list of all endpoints available on your Djangoldp-based server instance and\
+         the available methods, needed parameters and requests examples. The list of available endpoints depend on your instance configuration\
+             especially the list of bloxes and associated django models activated.',
+    'VERSION': '2.1.37',
+    'SERVE_INCLUDE_SCHEMA': False
+}
+
+############
+# SESSIONS #
+############
+
+# Same site policy
+DCS_SESSION_COOKIE_SAMESITE = 'none'
+
+##################
+# AUTHENTICATION #
+##################
+
+AUTHENTICATION_BACKENDS = ['django.contrib.auth.backends.ModelBackend', 'guardian.backends.ObjectPermissionBackend']
+
+OIDC_ACCESS_CONTROL_ALLOW_HEADERS = 'Content-Type, if-match, accept, authorization, DPoP, cache-control, pragma, prefer'
+
+# The minimum number of seconds a password reset link is valid for
+PASSWORD_RESET_TIMEOUT = 60 * 60 * 24 * 3
+
+DISABLE_LOCAL_OBJECT_FILTER = False
+
+GUARDIAN_AUTO_PREFETCH = True
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'

djangoldp/conf/ldpsettings.py

+import os
+import sys
+import yaml
+import logging
+from django.core.exceptions import ImproperlyConfigured
+from django.conf import settings as django_settings
+from pathlib import Path
+from collections import OrderedDict
+from typing import Iterable
+from importlib import import_module
+from . import default_settings
+
+
+logger = logging.getLogger(__name__)
+
+
+def configure(filename='settings.yml'):
+    """Helper function to configure django from LDPSettings."""
+
+    yaml_config = None
+    try:
+        with open(filename, 'r') as f:
+            yaml_config = yaml.safe_load(f)
+    except FileNotFoundError:
+        logger.info('Starting project without configuration file')
+
+    # ref: https://docs.djangoproject.com/fr/2.2/topics/settings/#custom-default-settings
+    ldpsettings = LDPSettings(yaml_config)
+
+    django_settings.configure(ldpsettings)
+
+
+class LDPSettings(object):
+
+    """Class managing the DjangoLDP configuration."""
+
+    def __init__(self, config):
+
+        """Build a Django Setting object from a dict."""
+
+        if django_settings.configured:
+            raise ImproperlyConfigured('Settings have been configured already')
+
+        self._config = config
+        self._explicit_settings = set(default_settings.__dict__.keys()) #default settings are explicit
+        self._settings = self.build_settings()
+
+    def build_settings(self, extend=['INSTALLED_APPS', 'MIDDLEWARE']):
+        """
+        Look for the parameters in multiple places.
+        Each step overrides the value of the previous key found. Except for "extend" list. Those value must be lists and all values found are added to these lists without managing duplications.
+
+        Resolution order of the configuration:
+          1. Core default settings
+          2. Packages settings
+          3. Code from a local settings.py file
+          4. YAML config file
+        """
+
+        # helper loop
+        def update_with(config):
+            for setting, value in config.items():
+                self._explicit_settings.add(setting)
+                if setting in extend:
+                    settings[setting].extend(value)
+
+                elif not setting.startswith('_'):
+                    settings.update({setting: value})
+
+        # start from default core settings
+        settings = default_settings.__dict__.copy()
+        logger.debug(f'Building settings from core defaults')
+
+        # INSTALLED_APPS starts empty
+        settings['INSTALLED_APPS'] = []
+
+        # look settings from packages in the order they are given (local overrides installed)
+        for pkg in self.DJANGOLDP_PACKAGES:
+
+            # FIXME: There is something better to do here with the sys.modules path
+            try:
+                # override with values from installed package
+                mod = import_module(f'{pkg}.djangoldp_settings')
+                update_with(mod.__dict__)
+                logger.debug(f'Updating settings from installed package {pkg}')
+            except ModuleNotFoundError:
+                pass
+
+            try:
+                # override with values from local package
+                mod = import_module(f'{pkg}.{pkg}.djangoldp_settings')
+                update_with(mod.__dict__)
+                logger.debug(f'Updating settings from local package {pkg}')
+            except ModuleNotFoundError:
+                pass
+
+        # look in settings.py file in directory
+        try:
+            mod = import_module('settings')
+            update_with(mod.__dict__)
+            logger.debug(f'Updating settings from local settings.py file')
+        except ModuleNotFoundError:
+            pass
+
+        # look in YAML config file 'server' section
+        try:
+            conf = self._config.get('server', {})
+            update_with(conf)
+            logger.debug(f'Updating settings with project config')
+        except KeyError:
+            pass
+
+        # In the end adds the INSTALLED_APPS from the core
+        settings['INSTALLED_APPS'].extend(getattr(default_settings,'INSTALLED_APPS'))
+
+        return settings
+
+    @property
+    def DJANGOLDP_PACKAGES(self):
+
+        """Returns the list of LDP packages configured."""
+
+        pkg = self._config.get('ldppackages', [])
+        return [] if pkg is None else pkg
+
+    @property
+    def INSTALLED_APPS(self):
+
+        """Return the installed apps and the LDP packages."""
+
+        # get ldp packages (they are django apps)
+        apps = self.DJANGOLDP_PACKAGES.copy()
+
+        # add the default apps
+        apps.extend(self._settings['INSTALLED_APPS'])
+
+        # As settings come from different origins duplicuation is likeliy to happen
+        return list(OrderedDict.fromkeys(apps))
+
+    def __getattr__(self, param):
+        """Return the requested parameter from cached settings."""
+        if param.startswith('_') or param.islower():
+            # raise the django exception for inexistent parameter
+            raise AttributeError(f'"{param}" is not compliant to django settings format')
+        try:
+            return self._settings[param]
+        except KeyError:
+            # raise the django exception for inexistent parameter
+            raise AttributeError(f'no "{param}" parameter found in settings')
+
+    def is_overridden(self, setting):
+        return setting in self._explicit_settings
\ No newline at end of file

djangoldp/conf/package_template/README.md

+# {{ app_name }}

djangoldp/conf/package_template/app_name/__init__.py-tpl

+__version__ = '0.0.0'

djangoldp/conf/package_template/app_name/admin.py-tpl

+from django.contrib import admin