From 828e846ead887be9465b7cdc0a13bfcbee2ca215 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Apr 2024 11:50:34 +0100
Subject: [PATCH 01/70] add RisEntry

---
 doajtest/unit/test_ris.py |  74 ++++++++++++++
 portality/lib/ris.py      | 205 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 279 insertions(+)
 create mode 100644 doajtest/unit/test_ris.py
 create mode 100644 portality/lib/ris.py

diff --git a/doajtest/unit/test_ris.py b/doajtest/unit/test_ris.py
new file mode 100644
index 000000000..7e2ef085b
--- /dev/null
+++ b/doajtest/unit/test_ris.py
@@ -0,0 +1,74 @@
+from unittest import TestCase
+
+from portality.lib.ris import RisEntry
+
+
+class TestRisEntry(TestCase):
+
+    def test_get_set_item__basic(self):
+        test_value = 'value_a'
+        entry = RisEntry()
+        entry['primary_author'] = test_value
+        assert entry['primary_author'] == test_value
+
+    def test_get_set_item__alias(self):
+        test_value = 'value_a'
+        entry = RisEntry()
+        entry['primary_author'] = test_value
+        assert entry['A1'] == test_value
+
+    def test_getitem__valid_undefined(self):
+        entry = RisEntry()
+        assert entry['primary_author'] is None
+
+    def test_setitem__raise_field_not_found(self):
+        entry = RisEntry()
+        with self.assertRaises(ValueError):
+            entry['qoidjqowijdkncoiqw'] = 'value_a'
+
+    def test_getitem__raise_field_not_found(self):
+        entry = RisEntry()
+        with self.assertRaises(ValueError):
+            print(entry['qoidjqowijdkncoiqw'])
+
+    def test_to_dict(self):
+        entry = RisEntry()
+        entry['primary_author'] = 'value_a'
+        entry['secondary_author'] = 'value_b'
+        entry['tertiary_author'] = 'value_c'
+        assert entry.to_dict() == {
+            'A1': 'value_a',
+            'A2': 'value_b',
+            'A3': 'value_c'
+        }
+
+    def test_to_text(self):
+        entry = RisEntry()
+        entry['primary_author'] = 'value_a'
+        entry['secondary_author'] = 'value_b'
+        entry['TY'] = 'JOUR'
+
+        expected = """
+TY  - JOUR
+A1  - value_a
+A2  - value_b
+ER  -
+        """.strip() + ' \n'
+
+        assert entry.to_text() == expected
+
+    def test_from_text(self):
+        expected = """
+        TY  - JOUR
+        A1  - value_a
+        A2  - value_b
+        ER  -
+                """.strip() + ' \n'
+
+        entry = RisEntry.from_text(expected)
+        assert entry['TY'] == 'JOUR'
+        assert dict(entry.to_dict()) == {
+            'TY': 'JOUR',
+            'A1': 'value_a',
+            'A2': 'value_b'
+        }
diff --git a/portality/lib/ris.py b/portality/lib/ris.py
new file mode 100644
index 000000000..f1ba0083d
--- /dev/null
+++ b/portality/lib/ris.py
@@ -0,0 +1,205 @@
+"""
+very simple library for RIS format
+
+file format references: https://en.wikipedia.org/wiki/RIS_(file_format)
+"""
+import logging
+from collections import OrderedDict
+from typing import Dict, Optional
+
+log = logging.getLogger(__name__)
+
+RTAG_TYPE = 'TY'
+RTAG_END = 'ER'
+RIS_ALIAS = [
+    ['A1', 'primary_author'],
+    ['A2', 'secondary_author'],
+    ['A3', 'tertiary_author'],
+    ['A4', 'quaternary_author'],
+    ['A5', 'quinary_author_compiler'],
+    ['A6', 'website_editor'],
+    ['AB', 'abstract_synopsis'],
+    ['AD', 'author_editor_address'],
+    ['AN', 'accession_number'],
+    ['AU', 'author_editor_translator'],
+    ['AV', 'availability_location'],
+    ['BT', 'primary_secondary_title'],
+    ['C1', 'custom1'],
+    ['C2', 'custom2'],
+    ['C3', 'custom3'],
+    ['C4', 'custom4'],
+    ['C5', 'custom5'],
+    ['C6', 'custom6'],
+    ['C7', 'custom7'],
+    ['C8', 'custom8'],
+    ['CA', 'caption'],
+    ['CL', 'classification'],
+    ['CN', 'call_number'],
+    ['CP', 'city_place_publication'],
+    ['CR', 'cited_references'],
+    ['CT', 'caption_primary_title'],
+    ['CY', 'place_published'],
+    ['DA', 'date'],
+    ['DB', 'name_of_database'],
+    ['DI', 'digital_object_identifier', 'di', ],
+    ['DO', 'digital_object_identifier2', 'do', ],
+    ['DOI', 'digital_object_identifier3', 'doi', ],
+    ['DP', 'database_provider'],
+    ['DS', 'data_source'],
+    ['ED', 'secondary_author'],
+    ['EP', 'end_page'],
+    ['ET', 'edition'],
+    ['FD', 'free_form_publication_data'],
+    ['H1', 'location_library'],
+    ['H2', 'location_call_number'],
+    ['ID', 'reference_identifier'],
+    ['IP', 'identifying_phrase'],
+    ['IS', 'number_volumes'],
+    ['J1', 'journal_abbreviation_1'],
+    ['J2', 'alternate_title'],
+    ['JA', 'journal_standard_abbreviation'],
+    ['JF', 'journal_full_name'],
+    ['JO', 'journal_abbreviation'],
+    ['K1', 'keyword1'],
+    ['KW', 'keyword_phrase'],
+    ['L1', 'file_attachments'],
+    ['L2', 'url_link'],
+    ['L3', 'doi_link'],
+    ['L4', 'figure_image_link'],
+    ['LA', 'language'],
+    ['LB', 'label'],
+    ['LK', 'links'],
+    ['LL', 'sponsoring_library_location'],
+    ['M1', 'miscellaneous1'],
+    ['M2', 'miscellaneous2'],
+    ['M3', 'type_of_work'],
+    ['N1', 'notes1'],
+    ['N2', 'abstract_notes'],
+    ['NO', 'notes'],
+    ['NV', 'number_of_volumes'],
+    ['OL', 'output_language'],
+    ['OP', 'original_publication'],
+    ['PA', 'personal_notes'],
+    ['PB', 'publisher'],
+    ['PMCID', 'pmcid'],
+    ['PMID', 'pmid'],
+    ['PP', 'place_of_publication'],
+    ['PY', 'publication_year'],
+    ['RD', 'retrieved_date'],
+    ['RI', 'reviewed_item'],
+    ['RN', 'research_notes'],
+    ['RP', 'reprint_status'],
+    ['RT', 'reference_type'],
+    ['SE', 'section'],
+    ['SF', 'subfile_database'],
+    ['SL', 'sponsoring_library'],
+    ['SN', 'issn_isbn'],
+    ['SP', 'start_pages'],
+    ['SR', 'source_type'],
+    ['ST', 'short_title'],
+    ['SV', 'series_volume'],
+    ['T1', 'primary_title'],
+    ['T2', 'secondary_title'],
+    ['T3', 'tertiary_title'],
+    ['TA', 'translated_author'],
+    ['TI', 'title'],
+    ['TT', 'translated_title'],
+    [RTAG_TYPE, 'type_of_reference'],
+    ['U1', 'user_definable1'],
+    ['U2', 'user_definable2'],
+    ['U3', 'user_definable3'],
+    ['U4', 'user_definable4'],
+    ['U5', 'user_definable5'],
+    ['U6', 'user_definable6'],
+    ['U7', 'user_definable7'],
+    ['U8', 'user_definable8'],
+    ['U9', 'user_definable9'],
+    ['U10', 'user_definable10'],
+    ['U11', 'user_definable11'],
+    ['U12', 'user_definable12'],
+    ['U13', 'user_definable13'],
+    ['U14', 'user_definable14'],
+    ['U15', 'user_definable15'],
+    ['UR', 'web_url'],
+    ['VL', 'volume'],
+    ['VO', 'volume_published_standard'],
+    ['WP', 'date_of_electronic_publication'],
+    ['WT', 'website_title'],
+    ['WV', 'website_version'],
+    ['Y1', 'year_date'],
+    ['Y2', 'access_date_secondary_date'],
+    ['YR', 'publication_year_ref']
+]
+
+
+def find_tag(field_name) -> Optional[str]:
+    for alias in RIS_ALIAS:
+        if field_name in alias:
+            return alias[0]
+    raise ValueError(f'Field not found: {field_name}')
+
+
+class RisEntry:
+
+    def __init__(self):
+        self.data: Dict[str, str] = OrderedDict()
+
+    def __setitem__(self, field_name, value):
+        tag = find_tag(field_name)
+        self.data[tag] = value
+
+    def __getitem__(self, field_name) -> str:
+        tag = find_tag(field_name)
+        return self.data.get(tag)
+
+    @classmethod
+    def from_dict(cls, d: dict):
+        instance = cls()
+        for k, v in d.items():
+            setattr(instance, k, v)
+        return instance
+
+    def to_dict(self) -> dict:
+        return self.data.copy()
+
+    @classmethod
+    def from_text(cls, text: str):
+        def _to_tag_value(line: str):
+            tag, value = line.split('-', 1)
+            tag = tag.strip()
+            value = value.lstrip()
+            value = value.replace('\\n', '\n')
+            return tag, value
+
+        text = text.strip()
+        lines = text.splitlines()
+        entry = RisEntry()
+        for line in lines:
+            tag, val = _to_tag_value(line)
+            if tag == RTAG_END:
+                break
+            entry[tag] = val
+        return entry
+
+    def to_text(self) -> str:
+        tags = list(self.data.keys())
+        if RTAG_TYPE in tags:
+            tags.remove(RTAG_TYPE)
+            tags.insert(0, RTAG_TYPE)
+
+        if RTAG_END in tags:
+            tags.remove(RTAG_END)
+
+        def _to_line(tag, value):
+            if '\n' in value:
+                value = value.replace('\n', '\\n')
+            if value is None:
+                value = ''
+            return f'{tag}  - {value}\n'
+
+        text = ''
+        for tag in tags:
+            text += _to_line(tag, self.data[tag])
+
+        text += _to_line(RTAG_END, '')
+        return text

From 4848dff4d45b5a114fb9964f91a52727c843729f Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Apr 2024 12:50:54 +0100
Subject: [PATCH 02/70] replace RIS_ALIAS

---
 doajtest/unit/test_ris.py |  18 +--
 portality/lib/ris.py      | 250 ++++++++++++++++++++------------------
 2 files changed, 138 insertions(+), 130 deletions(-)

diff --git a/doajtest/unit/test_ris.py b/doajtest/unit/test_ris.py
index 7e2ef085b..e49f5eea0 100644
--- a/doajtest/unit/test_ris.py
+++ b/doajtest/unit/test_ris.py
@@ -8,18 +8,18 @@ class TestRisEntry(TestCase):
     def test_get_set_item__basic(self):
         test_value = 'value_a'
         entry = RisEntry()
-        entry['primary_author'] = test_value
-        assert entry['primary_author'] == test_value
+        entry['A1'] = test_value
+        assert entry['A1'] == test_value
 
     def test_get_set_item__alias(self):
         test_value = 'value_a'
         entry = RisEntry()
-        entry['primary_author'] = test_value
+        entry['A1'] = test_value
         assert entry['A1'] == test_value
 
     def test_getitem__valid_undefined(self):
         entry = RisEntry()
-        assert entry['primary_author'] is None
+        assert entry['A1'] is None
 
     def test_setitem__raise_field_not_found(self):
         entry = RisEntry()
@@ -33,9 +33,9 @@ def test_getitem__raise_field_not_found(self):
 
     def test_to_dict(self):
         entry = RisEntry()
-        entry['primary_author'] = 'value_a'
-        entry['secondary_author'] = 'value_b'
-        entry['tertiary_author'] = 'value_c'
+        entry['A1'] = 'value_a'
+        entry['A2'] = 'value_b'
+        entry['A3'] = 'value_c'
         assert entry.to_dict() == {
             'A1': 'value_a',
             'A2': 'value_b',
@@ -44,8 +44,8 @@ def test_to_dict(self):
 
     def test_to_text(self):
         entry = RisEntry()
-        entry['primary_author'] = 'value_a'
-        entry['secondary_author'] = 'value_b'
+        entry['A1'] = 'value_a'
+        entry['A2'] = 'value_b'
         entry['TY'] = 'JOUR'
 
         expected = """
diff --git a/portality/lib/ris.py b/portality/lib/ris.py
index f1ba0083d..3c16c8379 100644
--- a/portality/lib/ris.py
+++ b/portality/lib/ris.py
@@ -11,131 +11,131 @@
 
 RTAG_TYPE = 'TY'
 RTAG_END = 'ER'
-RIS_ALIAS = [
-    ['A1', 'primary_author'],
-    ['A2', 'secondary_author'],
-    ['A3', 'tertiary_author'],
-    ['A4', 'quaternary_author'],
-    ['A5', 'quinary_author_compiler'],
-    ['A6', 'website_editor'],
-    ['AB', 'abstract_synopsis'],
-    ['AD', 'author_editor_address'],
-    ['AN', 'accession_number'],
-    ['AU', 'author_editor_translator'],
-    ['AV', 'availability_location'],
-    ['BT', 'primary_secondary_title'],
-    ['C1', 'custom1'],
-    ['C2', 'custom2'],
-    ['C3', 'custom3'],
-    ['C4', 'custom4'],
-    ['C5', 'custom5'],
-    ['C6', 'custom6'],
-    ['C7', 'custom7'],
-    ['C8', 'custom8'],
-    ['CA', 'caption'],
-    ['CL', 'classification'],
-    ['CN', 'call_number'],
-    ['CP', 'city_place_publication'],
-    ['CR', 'cited_references'],
-    ['CT', 'caption_primary_title'],
-    ['CY', 'place_published'],
-    ['DA', 'date'],
-    ['DB', 'name_of_database'],
-    ['DI', 'digital_object_identifier', 'di', ],
-    ['DO', 'digital_object_identifier2', 'do', ],
-    ['DOI', 'digital_object_identifier3', 'doi', ],
-    ['DP', 'database_provider'],
-    ['DS', 'data_source'],
-    ['ED', 'secondary_author'],
-    ['EP', 'end_page'],
-    ['ET', 'edition'],
-    ['FD', 'free_form_publication_data'],
-    ['H1', 'location_library'],
-    ['H2', 'location_call_number'],
-    ['ID', 'reference_identifier'],
-    ['IP', 'identifying_phrase'],
-    ['IS', 'number_volumes'],
-    ['J1', 'journal_abbreviation_1'],
-    ['J2', 'alternate_title'],
-    ['JA', 'journal_standard_abbreviation'],
-    ['JF', 'journal_full_name'],
-    ['JO', 'journal_abbreviation'],
-    ['K1', 'keyword1'],
-    ['KW', 'keyword_phrase'],
-    ['L1', 'file_attachments'],
-    ['L2', 'url_link'],
-    ['L3', 'doi_link'],
-    ['L4', 'figure_image_link'],
-    ['LA', 'language'],
-    ['LB', 'label'],
-    ['LK', 'links'],
-    ['LL', 'sponsoring_library_location'],
-    ['M1', 'miscellaneous1'],
-    ['M2', 'miscellaneous2'],
-    ['M3', 'type_of_work'],
-    ['N1', 'notes1'],
-    ['N2', 'abstract_notes'],
-    ['NO', 'notes'],
-    ['NV', 'number_of_volumes'],
-    ['OL', 'output_language'],
-    ['OP', 'original_publication'],
-    ['PA', 'personal_notes'],
-    ['PB', 'publisher'],
-    ['PMCID', 'pmcid'],
-    ['PMID', 'pmid'],
-    ['PP', 'place_of_publication'],
-    ['PY', 'publication_year'],
-    ['RD', 'retrieved_date'],
-    ['RI', 'reviewed_item'],
-    ['RN', 'research_notes'],
-    ['RP', 'reprint_status'],
-    ['RT', 'reference_type'],
-    ['SE', 'section'],
-    ['SF', 'subfile_database'],
-    ['SL', 'sponsoring_library'],
-    ['SN', 'issn_isbn'],
-    ['SP', 'start_pages'],
-    ['SR', 'source_type'],
-    ['ST', 'short_title'],
-    ['SV', 'series_volume'],
-    ['T1', 'primary_title'],
-    ['T2', 'secondary_title'],
-    ['T3', 'tertiary_title'],
-    ['TA', 'translated_author'],
-    ['TI', 'title'],
-    ['TT', 'translated_title'],
-    [RTAG_TYPE, 'type_of_reference'],
-    ['U1', 'user_definable1'],
-    ['U2', 'user_definable2'],
-    ['U3', 'user_definable3'],
-    ['U4', 'user_definable4'],
-    ['U5', 'user_definable5'],
-    ['U6', 'user_definable6'],
-    ['U7', 'user_definable7'],
-    ['U8', 'user_definable8'],
-    ['U9', 'user_definable9'],
-    ['U10', 'user_definable10'],
-    ['U11', 'user_definable11'],
-    ['U12', 'user_definable12'],
-    ['U13', 'user_definable13'],
-    ['U14', 'user_definable14'],
-    ['U15', 'user_definable15'],
-    ['UR', 'web_url'],
-    ['VL', 'volume'],
-    ['VO', 'volume_published_standard'],
-    ['WP', 'date_of_electronic_publication'],
-    ['WT', 'website_title'],
-    ['WV', 'website_version'],
-    ['Y1', 'year_date'],
-    ['Y2', 'access_date_secondary_date'],
-    ['YR', 'publication_year_ref']
+RIS_TAGS = [
+    'A1',  # primary_author
+    'A2',  # secondary_author
+    'A3',  # tertiary_author
+    'A4',  # quaternary_author
+    'A5',  # quinary_author_compiler
+    'A6',  # website_editor
+    'AB',  # abstract_synopsis
+    'AD',  # author_editor_address
+    'AN',  # accession_number
+    'AU',  # author_editor_translator
+    'AV',  # availability_location
+    'BT',  # primary_secondary_title
+    'C1',  # custom1
+    'C2',  # custom2
+    'C3',  # custom3
+    'C4',  # custom4
+    'C5',  # custom5
+    'C6',  # custom6
+    'C7',  # custom7
+    'C8',  # custom8
+    'CA',  # caption
+    'CL',  # classification
+    'CN',  # call_number
+    'CP',  # city_place_publication
+    'CR',  # cited_references
+    'CT',  # caption_primary_title
+    'CY',  # place_published
+    'DA',  # date
+    'DB',  # name_of_database
+    'DI',  # digital_object_identifier
+    'DO',  # digital_object_identifier2
+    'DOI',  # digital_object_identifier3
+    'DP',  # database_provider
+    'DS',  # data_source
+    'ED',  # secondary_author
+    'EP',  # end_page
+    'ET',  # edition
+    'FD',  # free_form_publication_data
+    'H1',  # location_library
+    'H2',  # location_call_number
+    'ID',  # reference_identifier
+    'IP',  # identifying_phrase
+    'IS',  # number_volumes
+    'J1',  # journal_abbreviation_1
+    'J2',  # alternate_title
+    'JA',  # journal_standard_abbreviation
+    'JF',  # journal_full_name
+    'JO',  # journal_abbreviation
+    'K1',  # keyword1
+    'KW',  # keyword_phrase
+    'L1',  # file_attachments
+    'L2',  # url_link
+    'L3',  # doi_link
+    'L4',  # figure_image_link
+    'LA',  # language
+    'LB',  # label
+    'LK',  # links
+    'LL',  # sponsoring_library_location
+    'M1',  # miscellaneous1
+    'M2',  # miscellaneous2
+    'M3',  # type_of_work
+    'N1',  # notes1
+    'N2',  # abstract_notes
+    'NO',  # notes
+    'NV',  # number_of_volumes
+    'OL',  # output_language
+    'OP',  # original_publication
+    'PA',  # personal_notes
+    'PB',  # publisher
+    'PMCID',  # pmcid
+    'PMID',  # pmid
+    'PP',  # place_of_publication
+    'PY',  # publication_year
+    'RD',  # retrieved_date
+    'RI',  # reviewed_item
+    'RN',  # research_notes
+    'RP',  # reprint_status
+    'RT',  # reference_type
+    'SE',  # section
+    'SF',  # subfile_database
+    'SL',  # sponsoring_library
+    'SN',  # issn_isbn
+    'SP',  # start_pages
+    'SR',  # source_type
+    'ST',  # short_title
+    'SV',  # series_volume
+    'T1',  # primary_title
+    'T2',  # secondary_title
+    'T3',  # tertiary_title
+    'TA',  # translated_author
+    'TI',  # title
+    'TT',  # translated_title
+    RTAG_TYPE,  # 'type_of_reference'
+    'U1',  # user_definable1
+    'U2',  # user_definable2
+    'U3',  # user_definable3
+    'U4',  # user_definable4
+    'U5',  # user_definable5
+    'U6',  # user_definable6
+    'U7',  # user_definable7
+    'U8',  # user_definable8
+    'U9',  # user_definable9
+    'U10',  # user_definable10
+    'U11',  # user_definable11
+    'U12',  # user_definable12
+    'U13',  # user_definable13
+    'U14',  # user_definable14
+    'U15',  # user_definable15
+    'UR',  # web_url
+    'VL',  # volume
+    'VO',  # volume_published_standard
+    'WP',  # date_of_electronic_publication
+    'WT',  # website_title
+    'WV',  # website_version
+    'Y1',  # year_date
+    'Y2',  # access_date_secondary_date
+    'YR',  # publication_year_ref
 ]
 
 
 def find_tag(field_name) -> Optional[str]:
-    for alias in RIS_ALIAS:
-        if field_name in alias:
-            return alias[0]
+    field_name = field_name.upper()
+    if field_name in RIS_TAGS:
+        return field_name
     raise ValueError(f'Field not found: {field_name}')
 
 
@@ -152,6 +152,14 @@ def __getitem__(self, field_name) -> str:
         tag = find_tag(field_name)
         return self.data.get(tag)
 
+    @property
+    def type(self):
+        return self[RTAG_TYPE]
+
+    @type.setter
+    def type(self, value):
+        self[RTAG_TYPE] = value
+
     @classmethod
     def from_dict(cls, d: dict):
         instance = cls()

From cd9b42abfb3d4eeadf1f7e2261b8ab2849b213f6 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Apr 2024 13:13:42 +0100
Subject: [PATCH 03/70] support values list

---
 doajtest/unit/test_ris.py | 38 +++++++++++++++-----------------------
 portality/lib/ris.py      | 32 +++++++++++++++++++++-----------
 2 files changed, 36 insertions(+), 34 deletions(-)

diff --git a/doajtest/unit/test_ris.py b/doajtest/unit/test_ris.py
index e49f5eea0..346181c79 100644
--- a/doajtest/unit/test_ris.py
+++ b/doajtest/unit/test_ris.py
@@ -5,21 +5,24 @@
 
 class TestRisEntry(TestCase):
 
-    def test_get_set_item__basic(self):
+    def test_get_set_item(self):
         test_value = 'value_a'
         entry = RisEntry()
         entry['A1'] = test_value
-        assert entry['A1'] == test_value
+        assert entry['A1'] == [test_value]
 
-    def test_get_set_item__alias(self):
-        test_value = 'value_a'
+    def test_append(self):
         entry = RisEntry()
-        entry['A1'] = test_value
-        assert entry['A1'] == test_value
+        entry.append('A1', '1')
+        entry['A1'].append('2')
+        assert entry['A1'] == ['1', '2']
+
+        entry['A1'] = '9'
+        assert entry['A1'] == ['9']
 
     def test_getitem__valid_undefined(self):
         entry = RisEntry()
-        assert entry['A1'] is None
+        assert entry['A1'] == []
 
     def test_setitem__raise_field_not_found(self):
         entry = RisEntry()
@@ -31,17 +34,6 @@ def test_getitem__raise_field_not_found(self):
         with self.assertRaises(ValueError):
             print(entry['qoidjqowijdkncoiqw'])
 
-    def test_to_dict(self):
-        entry = RisEntry()
-        entry['A1'] = 'value_a'
-        entry['A2'] = 'value_b'
-        entry['A3'] = 'value_c'
-        assert entry.to_dict() == {
-            'A1': 'value_a',
-            'A2': 'value_b',
-            'A3': 'value_c'
-        }
-
     def test_to_text(self):
         entry = RisEntry()
         entry['A1'] = 'value_a'
@@ -66,9 +58,9 @@ def test_from_text(self):
                 """.strip() + ' \n'
 
         entry = RisEntry.from_text(expected)
-        assert entry['TY'] == 'JOUR'
-        assert dict(entry.to_dict()) == {
-            'TY': 'JOUR',
-            'A1': 'value_a',
-            'A2': 'value_b'
+        assert entry.type == 'JOUR'
+        assert dict(entry.data) == {
+            'TY': ['JOUR'],
+            'A1': ['value_a'],
+            'A2': ['value_b'],
         }
diff --git a/portality/lib/ris.py b/portality/lib/ris.py
index 3c16c8379..3e012b51d 100644
--- a/portality/lib/ris.py
+++ b/portality/lib/ris.py
@@ -3,6 +3,7 @@
 
 file format references: https://en.wikipedia.org/wiki/RIS_(file_format)
 """
+import collections
 import logging
 from collections import OrderedDict
 from typing import Dict, Optional
@@ -142,19 +143,24 @@ def find_tag(field_name) -> Optional[str]:
 class RisEntry:
 
     def __init__(self):
-        self.data: Dict[str, str] = OrderedDict()
+        self.data: collections.defaultdict[str, list] = collections.defaultdict(list)
 
     def __setitem__(self, field_name, value):
         tag = find_tag(field_name)
-        self.data[tag] = value
+        self.data[tag] = [value]
 
-    def __getitem__(self, field_name) -> str:
+    def append(self, tag, value) -> list:
+        tag = find_tag(tag)
+        self[tag].append(value)
+        return self[tag]
+
+    def __getitem__(self, field_name) -> list:
         tag = find_tag(field_name)
-        return self.data.get(tag)
+        return self.data[tag]
 
     @property
     def type(self):
-        return self[RTAG_TYPE]
+        return self[RTAG_TYPE] and self[RTAG_TYPE][0]
 
     @type.setter
     def type(self, value):
@@ -164,11 +170,13 @@ def type(self, value):
     def from_dict(cls, d: dict):
         instance = cls()
         for k, v in d.items():
-            setattr(instance, k, v)
-        return instance
+            if isinstance(v, list):
+                for vv in v:
+                    instance[k].append(vv)
+            else:
+                instance[k].append(v)
 
-    def to_dict(self) -> dict:
-        return self.data.copy()
+        return instance
 
     @classmethod
     def from_text(cls, text: str):
@@ -186,7 +194,7 @@ def _to_tag_value(line: str):
             tag, val = _to_tag_value(line)
             if tag == RTAG_END:
                 break
-            entry[tag] = val
+            entry[tag].append(val)
         return entry
 
     def to_text(self) -> str:
@@ -207,7 +215,9 @@ def _to_line(tag, value):
 
         text = ''
         for tag in tags:
-            text += _to_line(tag, self.data[tag])
+            values = self.data[tag]
+            for v in values:
+                text += _to_line(tag, v)
 
         text += _to_line(RTAG_END, '')
         return text

From 5d07e34802119278b30000fcf8e6ad62be89300c Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Apr 2024 14:22:15 +0100
Subject: [PATCH 04/70] add type_of_reference

---
 portality/lib/ris.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/portality/lib/ris.py b/portality/lib/ris.py
index 3e012b51d..d2b741595 100644
--- a/portality/lib/ris.py
+++ b/portality/lib/ris.py
@@ -142,8 +142,9 @@ def find_tag(field_name) -> Optional[str]:
 
 class RisEntry:
 
-    def __init__(self):
+    def __init__(self, type_of_reference: str = None):
         self.data: collections.defaultdict[str, list] = collections.defaultdict(list)
+        self.type = type_of_reference
 
     def __setitem__(self, field_name, value):
         tag = find_tag(field_name)

From 2a8c3ebcf04791badb3ea9503667809420447155 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Apr 2024 14:22:39 +0100
Subject: [PATCH 05/70] add jsonpath_utils.py

---
 portality/lib/jsonpath_utils.py | 7 +++++++
 setup.py                        | 3 +++
 2 files changed, 10 insertions(+)
 create mode 100644 portality/lib/jsonpath_utils.py

diff --git a/portality/lib/jsonpath_utils.py b/portality/lib/jsonpath_utils.py
new file mode 100644
index 000000000..7201c9bfe
--- /dev/null
+++ b/portality/lib/jsonpath_utils.py
@@ -0,0 +1,7 @@
+from typing import Iterable
+
+import jsonpath_ng.ext
+
+
+def find_values(query: str, data: dict) -> Iterable:
+    return (m.value for m in jsonpath_ng.ext.parse(query).find(data))
diff --git a/setup.py b/setup.py
index eccafbada..a9a44680b 100644
--- a/setup.py
+++ b/setup.py
@@ -63,6 +63,9 @@
         'pandas~=2.0.1',  # pandas lets us generate URLs for linkcheck
         'gspread-dataframe~=3.3.1',
         'gspread-formatting~=1.1.2',
+
+        'jsonpath-ng~=1.6',
+
     ] + (["setproctitle==1.1.10"] if "linux" in sys.platform else []),
     extras_require={
         # prevent backtracking through all versions

From aa25ee3319d2a113f79e76f214491ee6e7bdf95c Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Apr 2024 14:22:59 +0100
Subject: [PATCH 06/70] add ArticleRisXWalk

---
 doajtest/unit/test_crosswalks_article_ris.py | 39 ++++++++++++++++++++
 portality/crosswalks/article_ris.py          | 35 ++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 doajtest/unit/test_crosswalks_article_ris.py
 create mode 100644 portality/crosswalks/article_ris.py

diff --git a/doajtest/unit/test_crosswalks_article_ris.py b/doajtest/unit/test_crosswalks_article_ris.py
new file mode 100644
index 000000000..b4b499f59
--- /dev/null
+++ b/doajtest/unit/test_crosswalks_article_ris.py
@@ -0,0 +1,39 @@
+import unittest
+
+from doajtest.fixtures import ArticleFixtureFactory
+from portality import models
+from portality.crosswalks.article_ris import ArticleRisXWalk
+
+
+class TestArticleRisXWalk(unittest.TestCase):
+    def test_article2ris(self):
+        article = ArticleFixtureFactory.make_article_source()
+        article = models.Article(**article)
+        article.bibjson().abstract = "abstract"
+        ris = ArticleRisXWalk.article2ris(article)
+        assert ris.type == 'JOUR'
+        assert ris['T1'] == [article.data['bibjson']['title']]
+        assert ris.to_text().split() == """
+TY  - JOUR
+T1  - Article Title
+AU  - The Author
+PY  - 1991
+JO  - The Title
+VL  - 1
+SP  - 3
+EP  - 21
+UR  - http://www.example.com/article
+AB  - abstract
+KW  - word
+KW  - key
+DOI  - 10.0000/SOME.IDENTIFIER
+ER  - 
+        """.split()
+
+    def test_article2ris__only_title(self):
+        ris = ArticleRisXWalk.article2ris({"bibjson": {"title": "Article Title"}})
+        assert ris.to_text().split() == """
+TY  - JOUR
+T1  - Article Title
+ER  - 
+        """.split()
diff --git a/portality/crosswalks/article_ris.py b/portality/crosswalks/article_ris.py
new file mode 100644
index 000000000..11006ab34
--- /dev/null
+++ b/portality/crosswalks/article_ris.py
@@ -0,0 +1,35 @@
+from typing import Union
+
+from portality import models
+from portality.lib import jsonpath_utils
+from portality.lib.ris import RisEntry
+
+RIS_ARTICLE_MAPPING = {
+    'T1': '$.bibjson.title',
+    'AU': '$.bibjson.author[*].name',
+    'PY': '$.bibjson.year',
+    'JO': '$.bibjson.journal.title',
+    'VL': '$.bibjson.journal.volume',
+    'SP': '$.bibjson.start_page',
+    'EP': '$.bibjson.end_page',
+    'UR': '$.bibjson.link[*].url',
+    'AB': '$.bibjson.abstract',
+    'KW': '$.bibjson.keywords[*]',
+    'DOI': '$.bibjson.identifier[?(@.type == "doi")].id',
+    'SN': '$.bibjson.journal.issns[*]',
+}
+
+
+class ArticleRisXWalk:
+
+    @classmethod
+    def article2ris(cls, article: Union[models.Article, dict]) -> RisEntry:
+        if isinstance(article, models.Article):
+            article = article.data
+
+        entry = RisEntry(type_of_reference='JOUR')
+        for tag, query in RIS_ARTICLE_MAPPING.items():
+            for v in jsonpath_utils.find_values(query, article):
+                entry[tag].append(v)
+
+        return entry

From 6dc56c697ecb59599036274775ba2cb47cf0b528 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Tue, 9 Apr 2024 10:32:57 +0100
Subject: [PATCH 07/70] fix type None

---
 portality/lib/ris.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/portality/lib/ris.py b/portality/lib/ris.py
index d2b741595..36bb83ab9 100644
--- a/portality/lib/ris.py
+++ b/portality/lib/ris.py
@@ -144,7 +144,8 @@ class RisEntry:
 
     def __init__(self, type_of_reference: str = None):
         self.data: collections.defaultdict[str, list] = collections.defaultdict(list)
-        self.type = type_of_reference
+        if type_of_reference:
+            self.type = type_of_reference
 
     def __setitem__(self, field_name, value):
         tag = find_tag(field_name)

From 50024ca3200ed6deda01be804db252a7be62fe4b Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Tue, 9 Apr 2024 10:36:11 +0100
Subject: [PATCH 08/70] add export ris for each article result

---
 portality/static/js/doaj.fieldrender.edges.js |  5 +++
 portality/view/doajservices.py                | 33 +++++++++++++++----
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js
index b82822cfb..6b670ca48 100644
--- a/portality/static/js/doaj.fieldrender.edges.js
+++ b/portality/static/js/doaj.fieldrender.edges.js
@@ -2872,6 +2872,8 @@ $.extend(true, doaj, {
                     published = 'Published ' + name;
                 }
 
+                const export_url = this.doaj_url + '/service/export/article/' + resultobj.id;
+
                 var frag = '<li class="card search-results__record">\
                     <article class="row">\
                       <div class="col-sm-8 search-results__main">\
@@ -2905,6 +2907,9 @@ $.extend(true, doaj, {
                           <li>\
                             <a href="' + this.doaj_url + '/toc/' + issns[0] + '" target="_blank" rel="noopener">About the journal</a>\
                           </li>\
+                          <li>\
+                            <a href="' + export_url + '" target="_blank">Export RIS</a>\
+                          </li>\
                           <li>\
                             ' + published + '\
                           </li>\
diff --git a/portality/view/doajservices.py b/portality/view/doajservices.py
index 48c9c6500..a46fd1461 100644
--- a/portality/view/doajservices.py
+++ b/portality/view/doajservices.py
@@ -1,13 +1,14 @@
-import json, urllib.request, urllib.parse, urllib.error, requests
+import json
+from io import BytesIO
 
-from flask import Blueprint, make_response, request, abort, render_template
+from flask import Blueprint, make_response, abort, render_template, send_file
 from flask_login import current_user, login_required
 
-from portality.core import app
-from portality.decorators import ssl_required, write_required, restrict_to_role
-from portality.util import jsonp
 from portality import lock, models
 from portality.bll import DOAJ
+from portality.crosswalks.article_ris import ArticleRisXWalk
+from portality.decorators import ssl_required, write_required
+from portality.util import jsonp
 
 blueprint = Blueprint('doajservices', __name__)
 
@@ -40,7 +41,7 @@ def unlock(object_type, object_id):
         abort(400)
 
     # otherwise, return success
-    resp = make_response(json.dumps({"result" : "success"}))
+    resp = make_response(json.dumps({"result": "success"}))
     resp.mimetype = "application/json"
     return resp
 
@@ -107,7 +108,8 @@ def group_status(group_id):
     :param group_id:
     :return:
     """
-    if (not (current_user.has_role("editor") and models.EditorGroup.pull(group_id).editor == current_user.id)) and (not current_user.has_role("admin")):
+    if (not (current_user.has_role("editor") and models.EditorGroup.pull(group_id).editor == current_user.id)) and (
+            not current_user.has_role("admin")):
         abort(404)
     svc = DOAJ.todoService()
     stats = svc.group_stats(group_id)
@@ -126,6 +128,7 @@ def dismiss_autocheck(autocheck_set_id, autocheck_id):
         abort(404)
     return make_response(json.dumps({"status": "success"}))
 
+
 @blueprint.route("/autocheck/undismiss/<autocheck_set_id>/<autocheck_id>", methods=["GET", "POST"])
 @jsonp
 @login_required
@@ -138,3 +141,19 @@ def undismiss_autocheck(autocheck_set_id, autocheck_id):
         abort(404)
     return make_response(json.dumps({"status": "success"}))
 
+
+@blueprint.route('/export/article/<article_id>')
+def export_article_ris(article_id):
+    article = models.Article.pull(article_id)
+    if not article:
+        abort(404)
+
+    byte_stream = BytesIO()
+    ris = ArticleRisXWalk.article2ris(article)
+    byte_stream.write(ris.to_text().encode('utf-8', errors='ignore'))
+    byte_stream.seek(0)
+
+    filename = f'article-{article_id[:10]}.ris'
+
+    resp = make_response(send_file(byte_stream, as_attachment=True, attachment_filename=filename))
+    return resp

From a839f54bc3373c94983b06b10dc14b6e73abde72 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Tue, 9 Apr 2024 12:49:20 +0100
Subject: [PATCH 09/70] default autocheck disabled for test cases

---
 doajtest/helpers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doajtest/helpers.py b/doajtest/helpers.py
index a8f33d52d..713d91989 100644
--- a/doajtest/helpers.py
+++ b/doajtest/helpers.py
@@ -129,6 +129,7 @@ class DoajTestCase(TestCase):
     @classmethod
     def create_app_patch(cls):
         return {
+            'AUTOCHECK_INCOMING': False,  # old test cases design and depend on work flow of autocheck disabled
             "STORE_IMPL": "portality.store.StoreLocal",
             "STORE_LOCAL_DIR": paths.rel2abs(__file__, "..", "tmp", "store", "main", cls.__name__.lower()),
             "STORE_TMP_DIR": paths.rel2abs(__file__, "..", "tmp", "store", "tmp", cls.__name__.lower()),

From 866c52c0ceef1f43a9b53929741371748baecf55 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Tue, 9 Apr 2024 12:52:59 +0100
Subject: [PATCH 10/70] avoid duplicate author names

---
 portality/crosswalks/article_ris.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/portality/crosswalks/article_ris.py b/portality/crosswalks/article_ris.py
index 11006ab34..2b57e2e03 100644
--- a/portality/crosswalks/article_ris.py
+++ b/portality/crosswalks/article_ris.py
@@ -4,9 +4,16 @@
 from portality.lib import jsonpath_utils
 from portality.lib.ris import RisEntry
 
+
+def extra_author_names(article) -> list:
+    query = '$.bibjson.author[*].name'
+    values = jsonpath_utils.find_values(query, article)
+    return sorted(set(values))
+
+
 RIS_ARTICLE_MAPPING = {
     'T1': '$.bibjson.title',
-    'AU': '$.bibjson.author[*].name',
+    'AU': extra_author_names,
     'PY': '$.bibjson.year',
     'JO': '$.bibjson.journal.title',
     'VL': '$.bibjson.journal.volume',
@@ -29,7 +36,12 @@ def article2ris(cls, article: Union[models.Article, dict]) -> RisEntry:
 
         entry = RisEntry(type_of_reference='JOUR')
         for tag, query in RIS_ARTICLE_MAPPING.items():
-            for v in jsonpath_utils.find_values(query, article):
+            if callable(query):
+                values = query(article)
+            else:
+                values = jsonpath_utils.find_values(query, article)
+
+            for v in values:
                 entry[tag].append(v)
 
         return entry

From 8f4fe0737e0034c2b363473e239ca8e52109b5fb Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 10 Apr 2024 10:39:53 +0100
Subject: [PATCH 11/70] add test_view_doajservices.py

---
 doajtest/unit/test_view_doajservices.py | 28 +++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doajtest/unit/test_view_doajservices.py

diff --git a/doajtest/unit/test_view_doajservices.py b/doajtest/unit/test_view_doajservices.py
new file mode 100644
index 000000000..3b41f39db
--- /dev/null
+++ b/doajtest/unit/test_view_doajservices.py
@@ -0,0 +1,28 @@
+from doajtest.fixtures import ArticleFixtureFactory
+from doajtest.helpers import DoajTestCase
+from portality.crosswalks.article_ris import ArticleRisXWalk
+from portality.models import Article
+from portality.util import url_for
+
+
+class TestDoajservices(DoajTestCase):
+
+    def test_export_article_ris(self):
+        article = Article(**ArticleFixtureFactory.make_article_source())
+        article.save(blocking=True)
+        Article.refresh()
+
+        ris = ArticleRisXWalk.article2ris(article).to_text()
+
+        with self.app_test.test_client() as t_client:
+            url = url_for('doajservices.export_article_ris', article_id=article.id)
+            response = t_client.get(url)
+            assert response.status_code == 200
+            assert response.get_data(as_text=True) == ris
+
+    def test_export_article_ris__not_found(self):
+        with self.app_test.test_client() as t_client:
+            url = url_for('doajservices.export_article_ris',
+                          article_id='article_id_that_does_not_exist')
+            response = t_client.get(url)
+            assert response.status_code == 404

From b761ad92aaddbe6d13bb56d53847a862c013f709 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 10 Apr 2024 11:19:04 +0100
Subject: [PATCH 12/70] improve RIS mapping

---
 portality/crosswalks/article_ris.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/portality/crosswalks/article_ris.py b/portality/crosswalks/article_ris.py
index 2b57e2e03..214720084 100644
--- a/portality/crosswalks/article_ris.py
+++ b/portality/crosswalks/article_ris.py
@@ -15,8 +15,10 @@ def extra_author_names(article) -> list:
     'T1': '$.bibjson.title',
     'AU': extra_author_names,
     'PY': '$.bibjson.year',
-    'JO': '$.bibjson.journal.title',
+    'JF': '$.bibjson.journal.title',
+    'PB': '$.bibjson.journal.publisher',
     'VL': '$.bibjson.journal.volume',
+    'IS': '$.bibjson.journal.number',
     'SP': '$.bibjson.start_page',
     'EP': '$.bibjson.end_page',
     'UR': '$.bibjson.link[*].url',
@@ -24,6 +26,7 @@ def extra_author_names(article) -> list:
     'KW': '$.bibjson.keywords[*]',
     'DOI': '$.bibjson.identifier[?(@.type == "doi")].id',
     'SN': '$.bibjson.journal.issns[*]',
+    'LA': '$.language.language',
 }
 
 

From 9ada58ee895617fe5e6c34a92f634cd22d805c0f Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 10 Apr 2024 11:24:23 +0100
Subject: [PATCH 13/70] add Export article in RIS format

---
 doajtest/testbook/public_site/public_search.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/doajtest/testbook/public_site/public_search.yml b/doajtest/testbook/public_site/public_search.yml
index 6ff91b38d..166b02c25 100644
--- a/doajtest/testbook/public_site/public_search.yml
+++ b/doajtest/testbook/public_site/public_search.yml
@@ -186,4 +186,13 @@ tests:
   - step: click spacebar to check the filter
     results:
       - filter is applied
-
+- title: Export article in RIS format
+  context:
+    role: anonymous
+  steps:
+  - step: Go to the DOAJ search page at /search/articles
+    results:
+      - Only articles are shown in the results
+  - step: Click on 'Export RIS' of any article
+    results:
+      - A RIS file is downloaded

From 3c138f1738e6fc6c3e67b730221664c131c9f14e Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 10 Apr 2024 18:32:52 +0100
Subject: [PATCH 14/70] fix testcases

---
 doajtest/unit/test_crosswalks_article_ris.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doajtest/unit/test_crosswalks_article_ris.py b/doajtest/unit/test_crosswalks_article_ris.py
index b4b499f59..5d58655f4 100644
--- a/doajtest/unit/test_crosswalks_article_ris.py
+++ b/doajtest/unit/test_crosswalks_article_ris.py
@@ -18,8 +18,10 @@ def test_article2ris(self):
 T1  - Article Title
 AU  - The Author
 PY  - 1991
-JO  - The Title
+JF  - The Title
+PB  - The Publisher
 VL  - 1
+IS  - 99
 SP  - 3
 EP  - 21
 UR  - http://www.example.com/article

From b0dfc6bca65c57a3cb8fda7993fa1391fdabe13b Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 15 Apr 2024 11:46:35 +0100
Subject: [PATCH 15/70] draft articleinfo

---
 portality/forms/application_forms.py |  6 ++++--
 portality/static/js/formulaic.js     | 19 +++++++++++++++++--
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/portality/forms/application_forms.py b/portality/forms/application_forms.py
index b370d2905..490269ad5 100644
--- a/portality/forms/application_forms.py
+++ b/portality/forms/application_forms.py
@@ -1881,7 +1881,8 @@ class FieldDefinitions:
         "entry_template": "application_form/_entry_group.html",
         "widgets": [
             {"infinite_repeat" : {"enable_on_repeat" : ["textarea"]}},
-            "note_modal"
+            "note_modal",
+            "article_info",
         ],
         "merge_disabled" : "merge_disabled_notes",
     }
@@ -3042,7 +3043,8 @@ def wtforms(field, settings):
     "trim_whitespace" : "formulaic.widgets.newTrimWhitespace",  # ~~-> TrimWhitespace:FormWidget~~
     "note_modal" : "formulaic.widgets.newNoteModal", # ~~-> NoteModal:FormWidget~~
     "autocheck": "formulaic.widgets.newAutocheck", # ~~-> Autocheck:FormWidget~~
-        "issn_link" : "formulaic.widgets.newIssnLink" # ~~-> IssnLink:FormWidget~~,
+    "issn_link" : "formulaic.widgets.newIssnLink", # ~~-> IssnLink:FormWidget~~,
+    "article_info": "formulaic.widgets.newArticleInfo", # ~~-> ArticleInfo:FormWidget~~
 }
 
 
diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js
index 0c5932c3c..7ae33d0a9 100644
--- a/portality/static/js/formulaic.js
+++ b/portality/static/js/formulaic.js
@@ -1155,14 +1155,14 @@ var formulaic = {
 
             this._renderAutocheck = function(autocheck) {
                 let frag = "<li>";
-                
+
                 if (autocheck.checked_by && doaj.autocheckers &&
                     doaj.autocheckers.registry.hasOwnProperty(autocheck.checked_by)) {
                     frag += (new doaj.autocheckers.registry[autocheck.checked_by]()).draw(autocheck)
                 } else {
                     frag += this._defaultRender(autocheck);
                 }
-                
+
                 frag += `</li>`;
                 return frag;
             }
@@ -2252,5 +2252,20 @@ var formulaic = {
 
             this.init();
         },
+
+        newArticleInfo : (params) => edges.instantiate(formulaic.widgets.ArticleInfo, params),
+        ArticleInfo: function ({formulaic, fieldDef, args}) {
+            const init = () => {
+                console.log()
+                console.log(this.fieldDef);
+                debugger
+            };
+
+            init();
+        },
+
+
+
+
     }
 };

From edcd98adb884e40eab5f4c86eeddea9054e9f402 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 15 Apr 2024 13:03:14 +0100
Subject: [PATCH 16/70] implement ArticleInfo

---
 portality/static/js/formulaic.js | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js
index 7ae33d0a9..ec438f321 100644
--- a/portality/static/js/formulaic.js
+++ b/portality/static/js/formulaic.js
@@ -2256,9 +2256,15 @@ var formulaic = {
         newArticleInfo : (params) => edges.instantiate(formulaic.widgets.ArticleInfo, params),
         ArticleInfo: function ({formulaic, fieldDef, args}) {
             const init = () => {
-                console.log()
-                console.log(this.fieldDef);
-                debugger
+                const paths = window.location.pathname.split('/')
+                const journalId = paths[paths.length - 1]
+                fetch(`/admin/journal/${journalId}/article-info`)
+                    .then(response => response.json())
+                    .then(data => {
+                        const $p = $('.doaj_seal__container').prev('p');
+                        const text = $p.text()
+                        $p.text(text + `This journal has ${data.n_articles} articles in DOAJ.`)
+                    })
             };
 
             init();

From c90cda05b647c8d1a042482aeee35b5570c9232c Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 15 Apr 2024 13:06:24 +0100
Subject: [PATCH 17/70] update dictionary

---
 docs/dictionary.md | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/docs/dictionary.md b/docs/dictionary.md
index 49ba798b8..fcbee273d 100644
--- a/docs/dictionary.md
+++ b/docs/dictionary.md
@@ -1,10 +1,11 @@
-| Short   | Description                  |
-|---------|------------------------------|
-| bgjob   | background job               |
-| noti    | notification                 |
-| noqa    | NO-QA (NO Quality Assurance) |
-| inst    | instance                     |
-| fmt     | format                       |
-| exparam | extra parameter              |
-| maned   | Managing Editor              |
-| gsheet  | Google Sheet                 |
\ No newline at end of file
+| Short    | Description                  |
+|----------|------------------------------|
+| bgjob    | background job               |
+| noti     | notification                 |
+| noqa     | NO-QA (NO Quality Assurance) |
+| inst     | instance                     |
+| fmt      | format                       |
+| exparam  | extra parameter              |
+| maned    | Managing Editor              |
+| gsheet   | Google Sheet                 |
+| svc,serv | service                      |
\ No newline at end of file

From dcd962af3b2c966f2a91d69e5495f80d9b03f47b Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 15 Apr 2024 13:06:53 +0100
Subject: [PATCH 18/70] setup article_info widgets

---
 portality/forms/application_forms.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/portality/forms/application_forms.py b/portality/forms/application_forms.py
index 490269ad5..71dde9389 100644
--- a/portality/forms/application_forms.py
+++ b/portality/forms/application_forms.py
@@ -1603,7 +1603,7 @@ class FieldDefinitions:
     # ~~->$ DOAJSeal:FormField~~
     DOAJ_SEAL = {
         "name": "doaj_seal",
-        "label": "The journal has fulfilled all the criteria for the Seal. Award the Seal?",
+        "label": "Award the Seal?",
         "input": "checkbox",
         "validate": [
             {
@@ -1626,7 +1626,10 @@ class FieldDefinitions:
                                 "the journal must use a persistent identifier"
                 }
             }
-        ]
+        ],
+        "widgets": [
+            "article_info",
+        ],
     }
 
     # FIXME: this probably shouldn't be in the admin form fieldsets, rather its own separate form
@@ -1882,7 +1885,6 @@ class FieldDefinitions:
         "widgets": [
             {"infinite_repeat" : {"enable_on_repeat" : ["textarea"]}},
             "note_modal",
-            "article_info",
         ],
         "merge_disabled" : "merge_disabled_notes",
     }

From 969c687d7d3eff26fcb6b2174f0358ea11a41c32 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 15 Apr 2024 13:44:21 +0100
Subject: [PATCH 19/70] implement query of journal_article_info

---
 portality/view/admin.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/portality/view/admin.py b/portality/view/admin.py
index 484441a02..187aee04e 100644
--- a/portality/view/admin.py
+++ b/portality/view/admin.py
@@ -7,13 +7,14 @@
 from flask_login import current_user, login_required
 from werkzeug.datastructures import MultiDict
 
-from portality import dao
 import portality.models as models
 from portality import constants
+from portality import dao
 from portality import lock
 from portality.background import BackgroundSummary
 from portality.bll import DOAJ, exceptions
 from portality.bll.exceptions import ArticleMergeConflict, DuplicateArticleException
+from portality.bll.services.query import Query
 from portality.core import app
 from portality.crosswalks.application_form import ApplicationFormXWalk
 from portality.decorators import ssl_required, restrict_to_role, write_required
@@ -28,8 +29,6 @@
 from portality.ui.messages import Messages
 from portality.util import flash_with_url, jsonp, make_json_resp, get_web_json_payload, validate_json
 from portality.view.forms import EditorGroupForm, MakeContinuation
-
-from portality.bll.services.query import Query
 from portality.view.view_helper import exparam_editing_user
 
 # ~~Admin:Blueprint~~
@@ -322,6 +321,15 @@ def journals_bulk_reinstate():
 #
 #####################################################################
 
+@blueprint.route("/journal/<journal_id>/article-info/", methods=["GET"])
+def journal_article_info(journal_id):
+    j = models.Journal.pull(journal_id)
+    if j is None:
+        abort(404)
+
+    return {'n_articles': models.Article.count_by_issns(j.bibjson().issns())}
+
+
 @blueprint.route("/journal/<journal_id>/continue", methods=["GET", "POST"])
 @login_required
 @ssl_required
@@ -432,7 +440,8 @@ def application(application_id):
                 flash(str(e))
                 return redirect(url_for("admin.application", application_id=ap.id, _anchor='cannot_edit'))
         else:
-            return fc.render_template(obj=ap, lock=lockinfo, form_diff=form_diff, current_journal=current_journal, lcc_tree=lcc_jstree, autochecks=autochecks)
+            return fc.render_template(obj=ap, lock=lockinfo, form_diff=form_diff, current_journal=current_journal,
+                                      lcc_tree=lcc_jstree, autochecks=autochecks)
 
 
 @blueprint.route("/application_quick_reject/<application_id>", methods=["POST"])

From 3dcc38ddc02e60684958383ed990a5ec4cff0803 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 15 Apr 2024 13:49:12 +0100
Subject: [PATCH 20/70] add checking before init

---
 portality/static/js/formulaic.js | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js
index ec438f321..07d986f98 100644
--- a/portality/static/js/formulaic.js
+++ b/portality/static/js/formulaic.js
@@ -2255,19 +2255,26 @@ var formulaic = {
 
         newArticleInfo : (params) => edges.instantiate(formulaic.widgets.ArticleInfo, params),
         ArticleInfo: function ({formulaic, fieldDef, args}) {
+            const sealSelector = '.doaj_seal__container'
+
             const init = () => {
                 const paths = window.location.pathname.split('/')
                 const journalId = paths[paths.length - 1]
                 fetch(`/admin/journal/${journalId}/article-info`)
                     .then(response => response.json())
                     .then(data => {
-                        const $p = $('.doaj_seal__container').prev('p');
+                        const $p = $(sealSelector).prev('p');
                         const text = $p.text()
                         $p.text(text + `This journal has ${data.n_articles} articles in DOAJ.`)
                     })
             };
 
-            init();
+
+            if ($(sealSelector).length) {
+                init();
+            } else {
+                console.log('skip ArticleInfo, seal section not found')
+            }
         },
 
 

From 96ac2eb3f8aa8fb26f05f6297b8d7f703590e0a2 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 15 Apr 2024 16:33:55 +0100
Subject: [PATCH 21/70] add test cases

---
 doajtest/fixtures/accounts.py                 |  6 ++-
 doajtest/helpers.py                           |  4 +-
 .../api_tests/test_api_crud_returnvalues.py   |  5 +--
 doajtest/unit/test_view_admin.py              | 37 +++++++++++++++++++
 portality/view/admin.py                       |  1 +
 5 files changed, 47 insertions(+), 6 deletions(-)
 create mode 100644 doajtest/unit/test_view_admin.py

diff --git a/doajtest/fixtures/accounts.py b/doajtest/fixtures/accounts.py
index a36dd60c6..4b974490f 100644
--- a/doajtest/fixtures/accounts.py
+++ b/doajtest/fixtures/accounts.py
@@ -83,7 +83,11 @@ def create_publisher_a():
     return publisher
 
 
-def create_maned_a():
+def create_maned_a(is_save=False):
     from portality import models
     maned = models.Account(**AccountFixtureFactory.make_managing_editor_source())
+    maned.set_password("password")
+    if is_save:
+        maned.save(blocking=True)
     return maned
+
diff --git a/doajtest/helpers.py b/doajtest/helpers.py
index a8f33d52d..0cd4fb76f 100644
--- a/doajtest/helpers.py
+++ b/doajtest/helpers.py
@@ -410,9 +410,9 @@ def assert_expected_dict(test_case: TestCase, target, expected: dict):
     test_case.assertDictEqual(actual, expected)
 
 
-def login(app_client, username, password, follow_redirects=True):
+def login(app_client, email, password, follow_redirects=True):
     return app_client.post(url_for('account.login'),
-                           data=dict(user=username, password=password),
+                           data=dict(user=email, password=password),
                            follow_redirects=follow_redirects)
 
 
diff --git a/doajtest/unit/api_tests/test_api_crud_returnvalues.py b/doajtest/unit/api_tests/test_api_crud_returnvalues.py
index 1d708b422..b398d0d23 100644
--- a/doajtest/unit/api_tests/test_api_crud_returnvalues.py
+++ b/doajtest/unit/api_tests/test_api_crud_returnvalues.py
@@ -1,3 +1,4 @@
+from doajtest import helpers
 from doajtest.helpers import DoajTestCase, with_es
 from portality import models
 from doajtest.fixtures import ApplicationFixtureFactory, ArticleFixtureFactory, JournalFixtureFactory
@@ -205,9 +206,7 @@ def test_04_article_structure_exceptions(self):
 
     @staticmethod
     def login(app, username, password):
-        return app.post('/account/login',
-                        data=dict(username=username, password=password),
-                        follow_redirects=True)
+        return helpers.login(app, username, password)
 
     @staticmethod
     def logout(app):
diff --git a/doajtest/unit/test_view_admin.py b/doajtest/unit/test_view_admin.py
new file mode 100644
index 000000000..2f111bb2d
--- /dev/null
+++ b/doajtest/unit/test_view_admin.py
@@ -0,0 +1,37 @@
+import json
+
+from doajtest import helpers
+from doajtest.fixtures import JournalFixtureFactory
+from doajtest.fixtures.accounts import create_maned_a
+from doajtest.helpers import DoajTestCase
+from portality import models
+from portality.util import url_for
+
+
+class TestViewAdmin(DoajTestCase):
+
+    def setUp(self):
+        super().setUp()
+        self.acc = create_maned_a(is_save=True)
+
+    def test_journal_article_info(self):
+        journal = models.Journal(
+            **JournalFixtureFactory.make_journal_source()
+        )
+        journal.save(blocking=True)
+        models.Journal.refresh()
+
+        with self.app_test.test_client() as client:
+            resp = helpers.login(client, self.acc.email, 'password')
+            assert resp.status_code == 200
+
+            resp = client.get(url_for("admin.journal_article_info", journal_id=journal.id))
+            assert resp.status_code == 200
+            assert json.loads(resp.data) == {'n_articles': 0}
+
+    def test_journal_article_info__not_found(self):
+        with self.app_test.test_client() as client:
+            helpers.login(client, self.acc.email, 'password')
+
+            resp = client.get(url_for("admin.journal_article_info", journal_id='aksjdlaksjdlkajsdlkajsdlk'))
+            assert resp.status_code == 404
diff --git a/portality/view/admin.py b/portality/view/admin.py
index 187aee04e..ba440cea7 100644
--- a/portality/view/admin.py
+++ b/portality/view/admin.py
@@ -322,6 +322,7 @@ def journals_bulk_reinstate():
 #####################################################################
 
 @blueprint.route("/journal/<journal_id>/article-info/", methods=["GET"])
+@login_required
 def journal_article_info(journal_id):
     j = models.Journal.pull(journal_id)
     if j is None:

From d2f234d6676e8f1f5a7033900046e565c0bdd756 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Fri, 19 Apr 2024 12:07:45 +0100
Subject: [PATCH 22/70] wording

---
 portality/templates/application_form/editorial_form_fields.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/portality/templates/application_form/editorial_form_fields.html b/portality/templates/application_form/editorial_form_fields.html
index 0b91ff624..f7d28bc3a 100644
--- a/portality/templates/application_form/editorial_form_fields.html
+++ b/portality/templates/application_form/editorial_form_fields.html
@@ -56,7 +56,7 @@ <h2 class="form__header">{{ fs.label }}</h2>
     {% set fs = formulaic_context.fieldset("seal") %}
     {% if fs %}
         <h2 class="form__header">{{ fs.label }}</h2>
-        <p>The journal has fulfilled all the criteria for the Seal. </p>
+        <p>The journal may have fulfilled all the criteria for the Seal. </p>
         {% for f in fs.fields() %}
             {% set field_template = f.template %}
             {%  include field_template %}

From 2ba0038dbea8822f4aca02cdb3b8631ff1b3986b Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 29 Apr 2024 14:02:19 +0100
Subject: [PATCH 23/70] add fmt

---
 doajtest/unit/test_view_doajservices.py       | 4 ++--
 portality/static/js/doaj.fieldrender.edges.js | 2 +-
 portality/static/vendor/edges                 | 2 +-
 portality/view/doajservices.py                | 8 ++++++--
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/doajtest/unit/test_view_doajservices.py b/doajtest/unit/test_view_doajservices.py
index 3b41f39db..30e41ba17 100644
--- a/doajtest/unit/test_view_doajservices.py
+++ b/doajtest/unit/test_view_doajservices.py
@@ -15,7 +15,7 @@ def test_export_article_ris(self):
         ris = ArticleRisXWalk.article2ris(article).to_text()
 
         with self.app_test.test_client() as t_client:
-            url = url_for('doajservices.export_article_ris', article_id=article.id)
+            url = url_for('doajservices.export_article_ris', article_id=article.id, fmt='ris')
             response = t_client.get(url)
             assert response.status_code == 200
             assert response.get_data(as_text=True) == ris
@@ -23,6 +23,6 @@ def test_export_article_ris(self):
     def test_export_article_ris__not_found(self):
         with self.app_test.test_client() as t_client:
             url = url_for('doajservices.export_article_ris',
-                          article_id='article_id_that_does_not_exist')
+                          article_id='article_id_that_does_not_exist', fmt='ris')
             response = t_client.get(url)
             assert response.status_code == 404
diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js
index 6b670ca48..3cd2d3559 100644
--- a/portality/static/js/doaj.fieldrender.edges.js
+++ b/portality/static/js/doaj.fieldrender.edges.js
@@ -2872,7 +2872,7 @@ $.extend(true, doaj, {
                     published = 'Published ' + name;
                 }
 
-                const export_url = this.doaj_url + '/service/export/article/' + resultobj.id;
+                const export_url = this.doaj_url + '/service/export/article/' + resultobj.id + '/ris';
 
                 var frag = '<li class="card search-results__record">\
                     <article class="row">\
diff --git a/portality/static/vendor/edges b/portality/static/vendor/edges
index 990f42201..9639b871a 160000
--- a/portality/static/vendor/edges
+++ b/portality/static/vendor/edges
@@ -1 +1 @@
-Subproject commit 990f4220163a3e18880f0bdc3ad5c80d234d22dd
+Subproject commit 9639b871acb1f6590ee78e236f2c9333479c9fe8
diff --git a/portality/view/doajservices.py b/portality/view/doajservices.py
index a46fd1461..251730d1a 100644
--- a/portality/view/doajservices.py
+++ b/portality/view/doajservices.py
@@ -142,12 +142,16 @@ def undismiss_autocheck(autocheck_set_id, autocheck_id):
     return make_response(json.dumps({"status": "success"}))
 
 
-@blueprint.route('/export/article/<article_id>')
-def export_article_ris(article_id):
+@blueprint.route('/export/article/<article_id>/<fmt>')
+def export_article_ris(article_id, fmt):
     article = models.Article.pull(article_id)
     if not article:
         abort(404)
 
+    if fmt != 'ris':
+        # only support ris for now
+        abort(404)
+
     byte_stream = BytesIO()
     ris = ArticleRisXWalk.article2ris(article)
     byte_stream.write(ris.to_text().encode('utf-8', errors='ignore'))

From 38fe7104cb99b8658414724389dc99aebc22de71 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 29 Apr 2024 14:09:23 +0100
Subject: [PATCH 24/70] add download icon

---
 .../static/doaj/images/feather-icons/download.svg    |  1 +
 portality/static/js/doaj.fieldrender.edges.js        | 12 +++++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100644 portality/static/doaj/images/feather-icons/download.svg

diff --git a/portality/static/doaj/images/feather-icons/download.svg b/portality/static/doaj/images/feather-icons/download.svg
new file mode 100644
index 000000000..76767a924
--- /dev/null
+++ b/portality/static/doaj/images/feather-icons/download.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-download"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path><polyline points="7 10 12 15 17 10"></polyline><line x1="12" y1="15" x2="12" y2="3"></line></svg>
\ No newline at end of file
diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js
index 3cd2d3559..276bb08c7 100644
--- a/portality/static/js/doaj.fieldrender.edges.js
+++ b/portality/static/js/doaj.fieldrender.edges.js
@@ -2899,8 +2899,7 @@ $.extend(true, doaj, {
                             <a href="' + ftl + '" target="_blank" rel="noopener"> Read online '
                 if (this.widget){
                     frag += '<img src="' + this.doaj_url + '/static/doaj/images/feather-icons/external-link.svg" alt="external-link icon">'
-                }
-                else {
+                } else {
                     frag += '<i data-feather="external-link" aria-hidden="true"></i>'
                 }
                 frag += '</a></li>\
@@ -2908,7 +2907,14 @@ $.extend(true, doaj, {
                             <a href="' + this.doaj_url + '/toc/' + issns[0] + '" target="_blank" rel="noopener">About the journal</a>\
                           </li>\
                           <li>\
-                            <a href="' + export_url + '" target="_blank">Export RIS</a>\
+                            <a href="' + export_url + '" target="_blank">\
+                            Export RIS '
+                if (this.widget){
+                    frag += '<img src="' + this.doaj_url + '/static/doaj/images/feather-icons/download.svg" alt="external-link icon">'
+                } else {
+                    frag += '<i data-feather="download" aria-hidden="true"></i>'
+                }
+                            frag += '</a>\
                           </li>\
                           <li>\
                             ' + published + '\

From ee0a321e8a0b100b5467ec7787696f2730030f6c Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 1 May 2024 12:42:41 +0100
Subject: [PATCH 25/70] link order

---
 portality/static/js/doaj.fieldrender.edges.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js
index 276bb08c7..fe135729c 100644
--- a/portality/static/js/doaj.fieldrender.edges.js
+++ b/portality/static/js/doaj.fieldrender.edges.js
@@ -2903,10 +2903,7 @@ $.extend(true, doaj, {
                     frag += '<i data-feather="external-link" aria-hidden="true"></i>'
                 }
                 frag += '</a></li>\
-                          <li>\
-                            <a href="' + this.doaj_url + '/toc/' + issns[0] + '" target="_blank" rel="noopener">About the journal</a>\
-                          </li>\
-                          <li>\
+                         <li>\
                             <a href="' + export_url + '" target="_blank">\
                             Export RIS '
                 if (this.widget){
@@ -2916,6 +2913,9 @@ $.extend(true, doaj, {
                 }
                             frag += '</a>\
                           </li>\
+                          <li>\
+                            <a href="' + this.doaj_url + '/toc/' + issns[0] + '" target="_blank" rel="noopener">About the journal</a>\
+                          </li>\
                           <li>\
                             ' + published + '\
                           </li>\

From 9532667c6405867abb01477f4119a6edcbc9c8d4 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 1 May 2024 13:15:01 +0100
Subject: [PATCH 26/70] naming

---
 doajtest/fixtures/accounts.py    | 4 ++--
 doajtest/unit/test_view_admin.py | 2 +-
 portality/static/vendor/edges    | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/doajtest/fixtures/accounts.py b/doajtest/fixtures/accounts.py
index 4b974490f..231c02ce3 100644
--- a/doajtest/fixtures/accounts.py
+++ b/doajtest/fixtures/accounts.py
@@ -83,11 +83,11 @@ def create_publisher_a():
     return publisher
 
 
-def create_maned_a(is_save=False):
+def create_maned_a(save=False):
     from portality import models
     maned = models.Account(**AccountFixtureFactory.make_managing_editor_source())
     maned.set_password("password")
-    if is_save:
+    if save:
         maned.save(blocking=True)
     return maned
 
diff --git a/doajtest/unit/test_view_admin.py b/doajtest/unit/test_view_admin.py
index 2f111bb2d..84eead555 100644
--- a/doajtest/unit/test_view_admin.py
+++ b/doajtest/unit/test_view_admin.py
@@ -12,7 +12,7 @@ class TestViewAdmin(DoajTestCase):
 
     def setUp(self):
         super().setUp()
-        self.acc = create_maned_a(is_save=True)
+        self.acc = create_maned_a(save=True)
 
     def test_journal_article_info(self):
         journal = models.Journal(
diff --git a/portality/static/vendor/edges b/portality/static/vendor/edges
index 990f42201..9639b871a 160000
--- a/portality/static/vendor/edges
+++ b/portality/static/vendor/edges
@@ -1 +1 @@
-Subproject commit 990f4220163a3e18880f0bdc3ad5c80d234d22dd
+Subproject commit 9639b871acb1f6590ee78e236f2c9333479c9fe8

From e52ca45d8e4937d78e7ba1fc1c14a06fafe509c6 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 1 May 2024 13:59:18 +0100
Subject: [PATCH 27/70] use multiple checkbox for styles only

---
 portality/crosswalks/journal_form.py | 4 ++--
 portality/forms/application_forms.py | 7 ++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/portality/crosswalks/journal_form.py b/portality/crosswalks/journal_form.py
index ecdc20c61..91ad74a13 100644
--- a/portality/crosswalks/journal_form.py
+++ b/portality/crosswalks/journal_form.py
@@ -289,7 +289,7 @@ def form2admin(cls, form, obj):
                 obj.set_editor(editor)
 
         if getattr(form, "doaj_seal", None):
-            obj.set_seal(form.doaj_seal.data)
+            obj.set_seal('y' in form.doaj_seal.data)
 
     @classmethod
     def bibjson2form(cls, bibjson, forminfo):
@@ -457,7 +457,7 @@ def admin2form(cls, obj, forminfo):
         if obj.editor is not None:
             forminfo['editor'] = obj.editor
 
-        forminfo['doaj_seal'] = obj.has_seal()
+        forminfo['doaj_seal'] = ['y'] if obj.has_seal() else []
 
 
 class JournalFormXWalk(JournalGenericXWalk):
diff --git a/portality/forms/application_forms.py b/portality/forms/application_forms.py
index d083a459f..93ee44e21 100644
--- a/portality/forms/application_forms.py
+++ b/portality/forms/application_forms.py
@@ -1603,8 +1603,13 @@ class FieldDefinitions:
     # ~~->$ DOAJSeal:FormField~~
     DOAJ_SEAL = {
         "name": "doaj_seal",
-        "label": "Award the Seal?",
+        "label": "The journal may have fulfilled all the criteria for the Seal.",
+        "multiple": True,
         "input": "checkbox",
+        "options": [
+            {"display": "Award the Seal?", "value": 'y'},
+        ],
+
         "validate": [
             {
                 "only_if" : {

From f415303a7e0b57a012a653c3f45fc788a186815b Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 1 May 2024 14:10:46 +0100
Subject: [PATCH 28/70] update checkbox title

---
 portality/static/js/formulaic.js                           | 7 +++----
 .../templates/application_form/editorial_form_fields.html  | 1 -
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js
index 07d986f98..04a6b85b8 100644
--- a/portality/static/js/formulaic.js
+++ b/portality/static/js/formulaic.js
@@ -2255,7 +2255,7 @@ var formulaic = {
 
         newArticleInfo : (params) => edges.instantiate(formulaic.widgets.ArticleInfo, params),
         ArticleInfo: function ({formulaic, fieldDef, args}) {
-            const sealSelector = '.doaj_seal__container'
+            const sealSelector = 'label[for=doaj_seal]';
 
             const init = () => {
                 const paths = window.location.pathname.split('/')
@@ -2263,9 +2263,8 @@ var formulaic = {
                 fetch(`/admin/journal/${journalId}/article-info`)
                     .then(response => response.json())
                     .then(data => {
-                        const $p = $(sealSelector).prev('p');
-                        const text = $p.text()
-                        $p.text(text + `This journal has ${data.n_articles} articles in DOAJ.`)
+                        const $ele = $(sealSelector);
+                        $ele.text($ele.text() + `This journal has ${data.n_articles} articles in DOAJ.`)
                     })
             };
 
diff --git a/portality/templates/application_form/editorial_form_fields.html b/portality/templates/application_form/editorial_form_fields.html
index ba8f74cab..e56769093 100644
--- a/portality/templates/application_form/editorial_form_fields.html
+++ b/portality/templates/application_form/editorial_form_fields.html
@@ -56,7 +56,6 @@ <h2 class="form__header">{{ fs.label }}</h2>
     {% set fs = formulaic_context.fieldset("seal") %}
     {% if fs %}
         <h2 class="form__header">{{ fs.label }}</h2>
-        <p>The journal may have fulfilled all the criteria for the Seal. </p>
         {% for f in fs.fields() %}
             {% set field_template = f.template %}
             {%  include field_template %}

From ee1b64226e2a5e8b32031150c30e7905cfef3e05 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 19 Jun 2024 11:09:35 +0100
Subject: [PATCH 29/70] fix for seal form field

---
 doajtest/fixtures/v2/common.py                                | 2 +-
 .../unit/application_processors/test_maned_journal_review.py  | 4 ++--
 portality/tasks/journal_bulk_edit.py                          | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doajtest/fixtures/v2/common.py b/doajtest/fixtures/v2/common.py
index efa5f7367..c550437c9 100644
--- a/doajtest/fixtures/v2/common.py
+++ b/doajtest/fixtures/v2/common.py
@@ -25,7 +25,7 @@
 }
 
 SEAL_FORM_EXPANDED = {
-    "doaj_seal": False,
+    "doaj_seal": [],
 }
 
 JOURNAL_LIKE_BIBJSON = {
diff --git a/doajtest/unit/application_processors/test_maned_journal_review.py b/doajtest/unit/application_processors/test_maned_journal_review.py
index 07badcbab..b62fbb146 100644
--- a/doajtest/unit/application_processors/test_maned_journal_review.py
+++ b/doajtest/unit/application_processors/test_maned_journal_review.py
@@ -148,7 +148,7 @@ def test_04_maned_review_doaj_seal(self):
         )
 
         # set the seal to False using the form
-        fc.form.doaj_seal.data = False
+        fc.form.doaj_seal.data = []
 
         # run the crosswalk, don't test it at all in this test
         fc.form2target()
@@ -162,7 +162,7 @@ def test_04_maned_review_doaj_seal(self):
         fc.source.set_seal(True)
         fc.source2form()
 
-        assert fc.form.doaj_seal.data is True
+        assert 'y' in fc.form.doaj_seal.data
 
     def test_05_maned_review_continuations(self):
         # construct it from form data (with a known source)
diff --git a/portality/tasks/journal_bulk_edit.py b/portality/tasks/journal_bulk_edit.py
index 3e39d4388..0b4242adb 100644
--- a/portality/tasks/journal_bulk_edit.py
+++ b/portality/tasks/journal_bulk_edit.py
@@ -123,8 +123,8 @@ def run(self):
                     job.add_audit_message("Setting {f} to {x} for journal {y}".format(f=k, x=v, y=journal_id))
                     fc.form[k].data = v
                 else:
-                    if v:
-                        fc.form.doaj_seal.data = v
+                    if v or (isinstance(v, str) and v.lower() == 'y'):
+                        fc.form.doaj_seal.data = ['y']
                 updated = True
 
             if note:

From 56a56ea473374c0242cf48f2cd5bac7ed38b9bbe Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Jul 2024 12:54:36 +0100
Subject: [PATCH 30/70] avoid send request by invalid id

---
 portality/static/js/formulaic.js | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js
index 04a6b85b8..5da5dca98 100644
--- a/portality/static/js/formulaic.js
+++ b/portality/static/js/formulaic.js
@@ -2255,25 +2255,24 @@ var formulaic = {
 
         newArticleInfo : (params) => edges.instantiate(formulaic.widgets.ArticleInfo, params),
         ArticleInfo: function ({formulaic, fieldDef, args}) {
-            const sealSelector = 'label[for=doaj_seal]';
-
-            const init = () => {
-                const paths = window.location.pathname.split('/')
-                const journalId = paths[paths.length - 1]
-                fetch(`/admin/journal/${journalId}/article-info`)
-                    .then(response => response.json())
-                    .then(data => {
-                        const $ele = $(sealSelector);
-                        $ele.text($ele.text() + `This journal has ${data.n_articles} articles in DOAJ.`)
-                    })
-            };
+            const $sealEle = $('label[for=doaj_seal]');
 
-
-            if ($(sealSelector).length) {
-                init();
-            } else {
+            if (!$sealEle.length) {
                 console.log('skip ArticleInfo, seal section not found')
+                return;
+            }
+
+            const idResult = window.location.pathname.match('/journal/([a-f0-9]+)')
+            if (!idResult) {
+                console.log('skip ArticleInfo, journal id not found')
+                return
             }
+            const journalId = idResult[1]
+            fetch(`/admin/journal/${journalId}/article-info`)
+                .then(response => response.json())
+                .then(data => {
+                    $sealEle.text($sealEle.text() + `This journal has ${data.n_articles} articles in DOAJ.`)
+                })
         },
 
 

From f768d1b069653ecf6ccfd8ad8d9abac742e3a843 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Jul 2024 13:04:30 +0100
Subject: [PATCH 31/70] support in_doaj for count_by_issns query

---
 portality/models/article.py | 10 +++++++---
 portality/view/admin.py     |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/portality/models/article.py b/portality/models/article.py
index d431bb5ae..392a16851 100644
--- a/portality/models/article.py
+++ b/portality/models/article.py
@@ -85,8 +85,8 @@ def find_by_issns(cls, issns):
         return articles
 
     @classmethod
-    def count_by_issns(cls, issns):
-        q = ArticleQuery(issns=issns)
+    def count_by_issns(cls, issns, in_doaj=None):
+        q = ArticleQuery(issns=issns, in_doaj=in_doaj)
         return cls.hit_count(q.query())
 
     @classmethod
@@ -866,9 +866,10 @@ class ArticleQuery(object):
     _issn_terms = { "terms" : {"index.issn.exact" : ["<list of issns here>"]} }
     _volume_term = { "term" : {"bibjson.journal.volume.exact" : "<volume here>"} }
 
-    def __init__(self, issns=None, volume=None):
+    def __init__(self, issns=None, volume=None, in_doaj=None):
         self.issns = issns
         self.volume = volume
+        self.in_doaj = in_doaj
 
     def query(self):
         q = deepcopy(self.base_query)
@@ -883,6 +884,9 @@ def query(self):
             vq["term"]["bibjson.journal.volume.exact"] = self.volume
             q["query"]["bool"]["must"].append(vq)
 
+        if self.in_doaj is not None:
+            q["query"]["bool"]["must"].append({"term": {"admin.in_doaj": self.in_doaj}})
+
         return q
     
 class ArticleVolumesQuery(object):
diff --git a/portality/view/admin.py b/portality/view/admin.py
index ba440cea7..72c4fb6a6 100644
--- a/portality/view/admin.py
+++ b/portality/view/admin.py
@@ -328,7 +328,7 @@ def journal_article_info(journal_id):
     if j is None:
         abort(404)
 
-    return {'n_articles': models.Article.count_by_issns(j.bibjson().issns())}
+    return {'n_articles': models.Article.count_by_issns(j.bibjson().issns(), in_doaj=True)}
 
 
 @blueprint.route("/journal/<journal_id>/continue", methods=["GET", "POST"])

From 5e9f946e232d33ef89b245874985893c5ca79bf3 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Jul 2024 13:08:23 +0100
Subject: [PATCH 32/70] change ArticleInfo result layout

---
 portality/static/js/formulaic.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js
index 5da5dca98..2068abe12 100644
--- a/portality/static/js/formulaic.js
+++ b/portality/static/js/formulaic.js
@@ -2255,7 +2255,7 @@ var formulaic = {
 
         newArticleInfo : (params) => edges.instantiate(formulaic.widgets.ArticleInfo, params),
         ArticleInfo: function ({formulaic, fieldDef, args}) {
-            const $sealEle = $('label[for=doaj_seal]');
+            const $sealEle = $('label[for=doaj_seal-0]');
 
             if (!$sealEle.length) {
                 console.log('skip ArticleInfo, seal section not found')
@@ -2271,7 +2271,7 @@ var formulaic = {
             fetch(`/admin/journal/${journalId}/article-info`)
                 .then(response => response.json())
                 .then(data => {
-                    $sealEle.text($sealEle.text() + `This journal has ${data.n_articles} articles in DOAJ.`)
+                    $sealEle.text($sealEle.text() + ` (This journal has ${data.n_articles} articles in DOAJ)`)
                 })
         },
 

From c8f3e734e6d27721184514241b253ef83d932401 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Mon, 8 Jul 2024 13:53:09 +0100
Subject: [PATCH 33/70] support admin_site_search redirection

---
 portality/static/js/formulaic.js |  7 ++++++-
 portality/view/admin.py          | 15 +++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js
index 2068abe12..66d918887 100644
--- a/portality/static/js/formulaic.js
+++ b/portality/static/js/formulaic.js
@@ -2271,7 +2271,12 @@ var formulaic = {
             fetch(`/admin/journal/${journalId}/article-info`)
                 .then(response => response.json())
                 .then(data => {
-                    $sealEle.text($sealEle.text() + ` (This journal has ${data.n_articles} articles in DOAJ)`)
+                    let articleText = `(This journal has ${data.n_articles} articles in DOAJ)`
+                    if (data.n_articles > 0) {
+                        const articlesUrl = `/admin/journal/${journalId}/article-info/admin-site-search`
+                        articleText = `<a href="${articlesUrl}">${articleText}</a>`
+                    }
+                    $sealEle.html($sealEle.text() + ` ${articleText}`)
                 })
         },
 
diff --git a/portality/view/admin.py b/portality/view/admin.py
index 72c4fb6a6..00d1edaec 100644
--- a/portality/view/admin.py
+++ b/portality/view/admin.py
@@ -331,6 +331,21 @@ def journal_article_info(journal_id):
     return {'n_articles': models.Article.count_by_issns(j.bibjson().issns(), in_doaj=True)}
 
 
+@blueprint.route("/journal/<journal_id>/article-info/admin-site-search", methods=["GET"])
+@login_required
+def journal_article_info_admin_site_search(journal_id):
+    j = models.Journal.pull(journal_id)
+    if j is None:
+        abort(404)
+
+    issns = j.bibjson().issns()
+    if not issns:
+        abort(404)
+
+    target_url = '/admin/admin_site_search?source={"query":{"bool":{"must":[{"term":{"admin.in_doaj":true}},{"term":{"es_type.exact":"article"}},{"query_string":{"query":"%s","default_operator":"AND","default_field":"index.issn.exact"}}]}},"track_total_hits":true}'
+    return redirect(target_url % issns[0].replace('-', r'\\-'))
+
+
 @blueprint.route("/journal/<journal_id>/continue", methods=["GET", "POST"])
 @login_required
 @ssl_required

From fd3f5eb972a52e5ffde9a036f145e7179c7b45cb Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Mon, 8 Jul 2024 15:59:34 +0100
Subject: [PATCH 34/70] update export text

---
 portality/static/js/doaj.fieldrender.edges.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js
index fe135729c..b667decf5 100644
--- a/portality/static/js/doaj.fieldrender.edges.js
+++ b/portality/static/js/doaj.fieldrender.edges.js
@@ -2905,7 +2905,7 @@ $.extend(true, doaj, {
                 frag += '</a></li>\
                          <li>\
                             <a href="' + export_url + '" target="_blank">\
-                            Export RIS '
+                            Export Citation (RIS) '
                 if (this.widget){
                     frag += '<img src="' + this.doaj_url + '/static/doaj/images/feather-icons/download.svg" alt="external-link icon">'
                 } else {

From a644d0c861ed6241e11bba8fe74c0e7e00894fd8 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Mon, 8 Jul 2024 16:36:42 +0100
Subject: [PATCH 35/70] fix language crosswalk mapping for ris

---
 portality/crosswalks/article_ris.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/portality/crosswalks/article_ris.py b/portality/crosswalks/article_ris.py
index 214720084..b9ee310ea 100644
--- a/portality/crosswalks/article_ris.py
+++ b/portality/crosswalks/article_ris.py
@@ -26,7 +26,7 @@ def extra_author_names(article) -> list:
     'KW': '$.bibjson.keywords[*]',
     'DOI': '$.bibjson.identifier[?(@.type == "doi")].id',
     'SN': '$.bibjson.journal.issns[*]',
-    'LA': '$.language.language',
+    'LA': '$.bibjson.journal.language[*]',
 }
 
 

From 3613ab278ec7b265a71a4faaa84bddeea325e047 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Thu, 18 Jul 2024 22:41:05 +0100
Subject: [PATCH 36/70] initial implementation (pending unit tests)

---
 .../testdrive/todo_maned_editor_associate.py  | 13 +++++-
 portality/bll/services/todo.py                | 44 ++++++++++++++++---
 portality/constants.py                        |  1 +
 portality/templates/dashboard/_todo.html      |  5 +++
 portality/templates/dashboard/index.html      |  6 +++
 portality/view/dashboard.py                   | 10 ++++-
 6 files changed, 69 insertions(+), 10 deletions(-)

diff --git a/doajtest/testdrive/todo_maned_editor_associate.py b/doajtest/testdrive/todo_maned_editor_associate.py
index 1fa8ff936..0ca722c55 100644
--- a/doajtest/testdrive/todo_maned_editor_associate.py
+++ b/doajtest/testdrive/todo_maned_editor_associate.py
@@ -142,6 +142,14 @@ def build_maned_applications(un, eg, owner, eponymous_group):
         "title": un + " Maned Pending Application"
     }]
 
+    app = build_application(un + " Maned On Hold Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD,
+                            editor_group=eg.name, owner=owner)
+    app.save()
+    apps["on_hold"] = [{
+        "id": app.id,
+        "title": un + " Maned On Hold Application"
+    }]
+
     app = build_application(un + " Maned Low Priority Pending Application", 1 * w, 1 * w,
                             constants.APPLICATION_STATUS_PENDING,
                             editor_group=eponymous_group.name, owner=owner)
@@ -154,11 +162,11 @@ def build_maned_applications(un, eg, owner, eponymous_group):
 
     lmur = build_application(un + " Last Month Maned Update Request", 5 * w, 5 * w, constants.APPLICATION_STATUS_UPDATE_REQUEST,
                            editor_group=eponymous_group.name, owner=owner, update_request=True)
-    lmur.save()
+    # lmur.save()
 
     tmur = build_application(un + " This Month Maned Update Request", 0, 0, constants.APPLICATION_STATUS_UPDATE_REQUEST,
                            editor_group=eponymous_group.name, owner=owner, update_request=True)
-    tmur.save()
+    # tmur.save()
 
     apps["update_request"] = [
         {
@@ -183,6 +191,7 @@ def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_grou
 
     if update_request:
         ap.application_type = constants.APPLICATION_TYPE_UPDATE_REQUEST
+        ap.set_current_journal(ap.makeid())
     else:
         ap.remove_current_journal()
         ap.remove_related_journal()
diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py
index fc57f66da..258094b0c 100644
--- a/portality/bll/services/todo.py
+++ b/portality/bll/services/todo.py
@@ -5,6 +5,7 @@
 from portality.lib import dates
 from datetime import datetime
 
+
 class TodoService(object):
     """
     ~~Todo:Service->DOAJ:Service~~
@@ -63,8 +64,7 @@ def group_stats(self, group_id):
 
         return stats
 
-
-    def top_todo(self, account, size=25, new_applications=True, update_requests=True):
+    def top_todo(self, account, size=25, new_applications=True, update_requests=True, on_hold=True):
         """
         Returns the top number of todo items for a given user
 
@@ -89,6 +89,8 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True
             if update_requests:
                 queries.append(TodoRules.maned_last_month_update_requests(size, maned_of))
                 queries.append(TodoRules.maned_new_update_requests(size, maned_of))
+            if on_hold:
+                queries.append(TodoRules.maned_on_hold(size, maned_of))
 
         if new_applications:    # editor and associate editor roles only deal with new applications
             if account.has_role("editor"):
@@ -174,7 +176,11 @@ def maned_stalled(cls, size, maned_of):
                 TodoQuery.is_new_application()
             ],
             must_nots=[
-                TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED])
+                TodoQuery.status([
+                    constants.APPLICATION_STATUS_ACCEPTED,
+                    constants.APPLICATION_STATUS_REJECTED,
+                    constants.APPLICATION_STATUS_ON_HOLD
+                ])
             ],
             sort=sort_date,
             size=size
@@ -191,7 +197,11 @@ def maned_follow_up_old(cls, size, maned_of):
                 TodoQuery.is_new_application()
             ],
             must_nots=[
-                TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED])
+                TodoQuery.status([
+                    constants.APPLICATION_STATUS_ACCEPTED,
+                    constants.APPLICATION_STATUS_REJECTED,
+                    constants.APPLICATION_STATUS_ON_HOLD
+                ])
             ],
             sort=sort_date,
             size=size
@@ -262,7 +272,11 @@ def maned_last_month_update_requests(cls, size, maned_of):
                 TodoQuery.is_update_request()
             ],
             must_nots=[
-                TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED])
+                TodoQuery.status([
+                    constants.APPLICATION_STATUS_ACCEPTED,
+                    constants.APPLICATION_STATUS_REJECTED,
+                    constants.APPLICATION_STATUS_ON_HOLD
+                ])
                 # TodoQuery.exists("admin.editor")
             ],
             sort=sort_date,
@@ -282,7 +296,11 @@ def maned_new_update_requests(cls, size, maned_of):
                 TodoQuery.is_update_request()
             ],
             must_nots=[
-                TodoQuery.status([constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED])
+                TodoQuery.status([
+                    constants.APPLICATION_STATUS_ACCEPTED,
+                    constants.APPLICATION_STATUS_REJECTED,
+                    constants.APPLICATION_STATUS_ON_HOLD
+                ])
                 # TodoQuery.exists("admin.editor")
             ],
             sort=sort_date,
@@ -290,6 +308,20 @@ def maned_new_update_requests(cls, size, maned_of):
         )
         return constants.TODO_MANED_NEW_UPDATE_REQUEST, assign_pending, sort_date, -2
 
+    @classmethod
+    def maned_on_hold(cls, size, maned_of):
+        sort_date = "created_date"
+        on_holds = TodoQuery(
+            musts=[
+                TodoQuery.editor_group(maned_of),
+                TodoQuery.is_new_application(),
+                TodoQuery.status([constants.APPLICATION_STATUS_ON_HOLD])
+            ],
+            sort=sort_date,
+            size=size
+        )
+        return constants.TODO_MANED_ON_HOLD, on_holds, sort_date, 0
+
     @classmethod
     def editor_stalled(cls, groups, size):
         sort_date = "created_date"
diff --git a/portality/constants.py b/portality/constants.py
index 2fec30877..991b1ec12 100644
--- a/portality/constants.py
+++ b/portality/constants.py
@@ -50,6 +50,7 @@
 TODO_MANED_ASSIGN_PENDING = "todo_maned_assign_pending"
 TODO_MANED_LAST_MONTH_UPDATE_REQUEST = "todo_maned_last_month_update_request"
 TODO_MANED_NEW_UPDATE_REQUEST = "todo_maned_new_update_request"
+TODO_MANED_ON_HOLD = "todo_maned_on_hold"
 TODO_EDITOR_STALLED = "todo_editor_stalled"
 TODO_EDITOR_FOLLOW_UP_OLD = "todo_editor_follow_up_old"
 TODO_EDITOR_COMPLETED = "todo_editor_completed"
diff --git a/portality/templates/dashboard/_todo.html b/portality/templates/dashboard/_todo.html
index 069ba8963..7303003ea 100644
--- a/portality/templates/dashboard/_todo.html
+++ b/portality/templates/dashboard/_todo.html
@@ -41,6 +41,11 @@
             "feather": "edit",
             "show_status": true
         },
+        constants.TODO_MANED_ON_HOLD: {
+            "text" : "<strong>On Hold Application</strong> <span class='todo-list__status'>Review status</span>",
+            "colour" : "var(--sanguine)",
+            "feather": "x-circle"
+        },
         constants.TODO_EDITOR_STALLED: {
             "text" : "<strong>Stalled</strong> <span class='todo-list__status'>Chase Associate Editor</span>",
             "show_status": true,
diff --git a/portality/templates/dashboard/index.html b/portality/templates/dashboard/index.html
index 99459988b..c2ba80a21 100644
--- a/portality/templates/dashboard/index.html
+++ b/portality/templates/dashboard/index.html
@@ -21,6 +21,12 @@
         {% else %}
             <span class="tag tag--tertiary"><a href="{{ url_for("dashboard.top_todo") }}?filter=ur">Update Requests</a></span>
         {% endif %}
+
+        {%  if request.values.get("filter") == "oh" %}
+            <span class="tag tag--primary"><a href="{{ url_for("dashboard.top_todo") }}?filter=oh">On Hold</a></span>
+        {% else %}
+            <span class="tag tag--tertiary"><a href="{{ url_for("dashboard.top_todo") }}?filter=oh">On Hold</a></span>
+        {% endif %}
     </div>
   {% include "dashboard/_todo.html" %}
   <section>
diff --git a/portality/view/dashboard.py b/portality/view/dashboard.py
index 2f63949a1..5637cc726 100644
--- a/portality/view/dashboard.py
+++ b/portality/view/dashboard.py
@@ -19,10 +19,15 @@
 @ssl_required
 def top_todo():
     filter = request.values.get("filter")
-    new_applications, update_requests = True, True
+    new_applications, update_requests, on_hold = True, True, True
     if filter == "na":
+        on_hold = False
         update_requests = False
     elif filter == "ur":
+        on_hold = False
+        new_applications = False
+    elif filter == "oh":
+        update_requests = False
         new_applications = False
 
     # ~~-> Todo:Service~~
@@ -30,7 +35,8 @@ def top_todo():
     todos = svc.top_todo(current_user._get_current_object(),
                          size=app.config.get("TODO_LIST_SIZE"),
                          new_applications=new_applications,
-                         update_requests=update_requests)
+                         update_requests=update_requests,
+                         on_hold=on_hold)
 
     # ~~-> Dashboard:Page~~
     return render_template('dashboard/index.html', todos=todos)

From e7a270979fcf7e4731067999428a2d850ed63655 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Fri, 19 Jul 2024 08:42:00 +0100
Subject: [PATCH 37/70] add unit testing for on hold dashboard

---
 .../bll_todo_maned/top_todo_maned.matrix.csv  | 12 +--
 .../top_todo_maned.settings.csv               | 76 ++++++++++---------
 .../top_todo_maned.settings.json              | 47 ++++++++++++
 doajtest/unit/test_bll_todo_top_todo_maned.py |  6 +-
 4 files changed, 98 insertions(+), 43 deletions(-)

diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv b/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv
index 9965c8049..0e0a97573 100644
--- a/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv
+++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.matrix.csv
@@ -1,6 +1,6 @@
-test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order
-1,none,ArgumentException,0,0,0,0,0,0,,,,,,
-2,no_role,,0,0,0,0,0,0,,,,,,
-3,admin,,1,1,1,1,1,1,1,2,3,4,5,6
-4,editor,,0,0,0,0,0,0,,,,,,
-5,assed,,0,0,0,0,0,0,,,,,,
+test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_on_hold,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order,todo_maned_on_hold_order
+1,none,ArgumentException,0,0,0,0,0,0,0,,,,,,,
+2,no_role,,0,0,0,0,0,0,0,,,,,,,
+3,admin,,1,1,1,1,1,1,1,1,2,3,4,5,6,7
+4,editor,,0,0,0,0,0,0,0,,,,,,,
+5,assed,,0,0,0,0,0,0,0,,,,,,,
diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv
index a8148032f..3d0f77656 100644
--- a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv
+++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.csv
@@ -1,36 +1,40 @@
-field,test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order
-type,index,generated,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional
-default,,,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,
-values,,none,ArgumentException,,,,,,,,,,,,
-values,,no_role,,,,,,,,,,,,,
-values,,admin,,,,,,,,,,,,,
-values,,editor,,,,,,,,,,,,,
-values,,assed,,,,,,,,,,,,,
-,,,,,,,,,,,,,,,
-conditional raises,,none,ArgumentException,,,,,,,,,,,,
-,,,,,,,,,,,,,,,
-conditional todo_maned_stalled,,admin,,1,,,,,,,,,,,
-conditional todo_maned_stalled,,!admin,,0,,,,,,,,,,,
-,,,,,,,,,,,,,,,
-conditional todo_maned_follow_up_old,,admin,,,1,,,,,,,,,,
-conditional todo_maned_follow_up_old,,!admin,,,0,,,,,,,,,,
-,,,,,,,,,,,,,,,
-conditional todo_maned_ready,,admin,,,,1,,,,,,,,,
-conditional todo_maned_ready,,!admin,,,,0,,,,,,,,,
-,,,,,,,,,,,,,,,
-conditional todo_maned_completed,,admin,,,,,1,,,,,,,,
-conditional todo_maned_completed,,!admin,,,,,0,,,,,,,,
-,,,,,,,,,,,,,,,
-conditional todo_maned_assign_pending,,admin,,,,,,1,,,,,,,
-conditional todo_maned_assign_pending,,!admin,,,,,,0,,,,,,,
-,,,,,,,,,,,,,,,
-conditional todo_maned_new_update_request,,admin,,,,,,,1,,,,,,
-conditional todo_maned_new_update_request,,!admin,,,,,,,0,,,,,,
-,,,,,,,,,,,,,,,
-conditional todo_maned_new_update_request_order,,admin,,,,,,,,1,,,,,
-conditional todo_maned_ready_order,,admin,,,,,,,,,2,,,,
-conditional todo_maned_follow_up_old_order,,admin,,,,,,,,,,3,,,
-conditional todo_maned_stalled_order,,admin,,,,,,,,,,,4,,
-conditional todo_maned_assign_pending_order,,admin,,,,,,,,,,,,5,
-conditional todo_maned_completed_order,,admin,,,,,,,,,,,,,6
\ No newline at end of file
+field,test_id,account,raises,todo_maned_stalled,todo_maned_follow_up_old,todo_maned_ready,todo_maned_completed,todo_maned_assign_pending,todo_maned_new_update_request,todo_maned_on_hold,todo_maned_new_update_request_order,todo_maned_ready_order,todo_maned_follow_up_old_order,todo_maned_stalled_order,todo_maned_assign_pending_order,todo_maned_completed_order,todo_maned_on_hold_order
+type,index,generated,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional,conditional
+default,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+values,,none,ArgumentException,,,,,,,,,,,,,,
+values,,no_role,,,,,,,,,,,,,,,
+values,,admin,,,,,,,,,,,,,,,
+values,,editor,,,,,,,,,,,,,,,
+values,,assed,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional raises,,none,ArgumentException,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional todo_maned_stalled,,admin,,1,,,,,,,,,,,,,
+conditional todo_maned_stalled,,!admin,,0,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional todo_maned_follow_up_old,,admin,,,1,,,,,,,,,,,,
+conditional todo_maned_follow_up_old,,!admin,,,0,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional todo_maned_ready,,admin,,,,1,,,,,,,,,,,
+conditional todo_maned_ready,,!admin,,,,0,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional todo_maned_completed,,admin,,,,,1,,,,,,,,,,
+conditional todo_maned_completed,,!admin,,,,,0,,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional todo_maned_assign_pending,,admin,,,,,,1,,,,,,,,,
+conditional todo_maned_assign_pending,,!admin,,,,,,0,,,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional todo_maned_new_update_request,,admin,,,,,,,1,,,,,,,,
+conditional todo_maned_new_update_request,,!admin,,,,,,,0,,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional todo_maned_on_hold,,admin,,,,,,,,1,,,,,,,
+conditional todo_maned_on_hold,,!admin,,,,,,,,0,,,,,,,
+,,,,,,,,,,,,,,,,,
+conditional todo_maned_new_update_request_order,,admin,,,,,,,,,1,,,,,,
+conditional todo_maned_ready_order,,admin,,,,,,,,,,2,,,,,
+conditional todo_maned_follow_up_old_order,,admin,,,,,,,,,,,3,,,,
+conditional todo_maned_stalled_order,,admin,,,,,,,,,,,,4,,,
+conditional todo_maned_assign_pending_order,,admin,,,,,,,,,,,,,5,,
+conditional todo_maned_completed_order,,admin,,,,,,,,,,,,,,6,
+conditional todo_maned_on_hold_order,,admin,,,,,,,,,,,,,,,7
\ No newline at end of file
diff --git a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json
index 6625f298f..a7ef53dc1 100644
--- a/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json
+++ b/doajtest/matrices/bll_todo_maned/top_todo_maned.settings.json
@@ -207,6 +207,35 @@
         }
       }
     },
+    {
+      "name": "todo_maned_on_hold",
+      "type": "conditional",
+      "default": "",
+      "values": {
+        "1": {
+          "conditions": [
+            {
+              "account": {
+                "or": [
+                  "admin"
+                ]
+              }
+            }
+          ]
+        },
+        "0": {
+          "conditions": [
+            {
+              "account": {
+                "nor": [
+                  "admin"
+                ]
+              }
+            }
+          ]
+        }
+      }
+    },
     {
       "name": "todo_maned_new_update_request_order",
       "type": "conditional",
@@ -314,6 +343,24 @@
           ]
         }
       }
+    },
+    {
+      "name": "todo_maned_on_hold_order",
+      "type": "conditional",
+      "default": "",
+      "values": {
+        "7": {
+          "conditions": [
+            {
+              "account": {
+                "or": [
+                  "admin"
+                ]
+              }
+            }
+          ]
+        }
+      }
     }
   ]
 }
\ No newline at end of file
diff --git a/doajtest/unit/test_bll_todo_top_todo_maned.py b/doajtest/unit/test_bll_todo_top_todo_maned.py
index 5322a8c9a..1c5ae4457 100644
--- a/doajtest/unit/test_bll_todo_top_todo_maned.py
+++ b/doajtest/unit/test_bll_todo_top_todo_maned.py
@@ -41,7 +41,8 @@ def test_top_todo(self, name, kwargs):
             "todo_maned_ready",
             "todo_maned_completed",
             "todo_maned_assign_pending",
-            "todo_maned_new_update_request"
+            "todo_maned_new_update_request",
+            "todo_maned_on_hold"
         ]
 
         category_args = {
@@ -100,6 +101,9 @@ def assign_pending(ap):
         self.build_application("maned_update_request", 5 * w, 5 * w, constants.APPLICATION_STATUS_UPDATE_REQUEST, apps,
                                update_request=True)
 
+        # an application that was modifed recently into the ready status (todo_maned_completed)
+        self.build_application("maned_on_hold", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD, apps)
+
         # Applications that should never be reported
         ############################################
 

From 5bf71ad623d0c3fa15a6a1ff4113fe3abf21e8a8 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Fri, 19 Jul 2024 10:23:38 +0100
Subject: [PATCH 38/70] update maned todo functional test script

---
 doajtest/testbook/dashboards/maned_todo.yml | 23 +++++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/doajtest/testbook/dashboards/maned_todo.yml b/doajtest/testbook/dashboards/maned_todo.yml
index 96c7b171c..017965bc7 100644
--- a/doajtest/testbook/dashboards/maned_todo.yml
+++ b/doajtest/testbook/dashboards/maned_todo.yml
@@ -27,7 +27,7 @@ tests:
       - step: Go to the maned dashboard page
         path: /dashboard
         results:
-          - You can see 16 applications in your priority list
+          - You can see 17 applications in your priority list
           - Your priority list contains a mixture of managing editor items (actions related to teams you are the managing editor for), 
             editor items (actions related to teams you are the editor for) and associate items (actions related to applications which
             are assigned specifically to you for review).
@@ -37,30 +37,31 @@ tests:
           - At least one of your priority items is for an application in the state ready (it should indicate that it is for your maned group)
           - At least one of your priority items is for an application in the completed state which has not been updated for more than 2 weeks (it should indicate that it is for your maned group)
           - At least one of your priority items is for an application in the pending state which has not been updated for more than 2 weeks (it should indicate that it is for your maned group)
+          - At least one of your priority items is for an application in the "on hold" state
           - Your lowest priority item (last in the list) is for an update request which was submitted this month
       - step: click on the managing editor's ready application
       - step: Change the application status to "Accepted" and save
       - step: close the tab, return to the dashboard and reload the page
         results:
-          - You can see 15 applications in your priority list
+          - You can see 16 applications in your priority list
           - The application you have just edited has disappeared from your priority list
       - step: click on the [in progress] stalled managing editor's application
       - step: make any minor adjustment to the metadata and save
       - step: close the tab, return to the dashboard and reload the page
         results:
-          - You can see 14 applications in your priority list
+          - You can see 15 applications in your priority list
           - The application you just edited has disappeared from your priority list
       - step: click on the "completed" maned application
       - step: Change the application to "ready" status
       - step: close the tab, return to the dashboard and reload the page
         results:
-          - You can still see 14 applications in your priority list
+          - You can still see 13 applications in your priority list
           - The completed application you just moved to ready is now in your priority list as a ready application
       - step: click on the pending managing editor's application
       - step: Assign the item to an editor in the selected group (there should be a test editor available to you to select)
       - step: close the tab, return to the dashboard and reload the page
         results:
-          - You have 13 applications left in your todo list
+          - You have 12 applications left in your todo list
           - The pending application you just edited is no longer visible
 
   - title: Filtering the todo list
@@ -74,22 +75,26 @@ tests:
       - step: Go to the maned dashboard page
         path: /dashboard
         results:
-          - You can see 16 applications in your priority list
+          - You can see 17 applications in your priority list
           - Your highest priority item (first in the list) is for an update request which was submitted last month
           - Your lowest priority item (last in the list) is for an update request which was submitted this month
-          - On the top right of the todo list are a set of filter buttons "Show all", "New Applications" and "Update Requests"
+          - On the top right of the todo list are a set of filter buttons "Show all", "New Applications", "Update Requests" and "On Hold"
           - The "Show all" button is highlighted
       - step: click on the "New Applications" filter button
         results:
           - You can see 14 applications in your priority list
-          - The update requests which were on the previous screen are no longer visible
+          - The update requests and "on hold" items which were on the previous screen are no longer visible
           - The "New Applications" filter button is now highlighted
       - step: click on the "Update Request" filter button
         results:
-          - You can see 12application in your priority list
+          - You can see 2 applications in your priority list
           - Your highest priority item (first in the list) is for an update request which was submitted last month
           - Your lowest priority item (last in the list) is for an update request which was submitted this month
           - The "Update Request" filter button is now highlighted
+      - step: click on the "On Hold" filter button
+        results:
+          - You can see 1 application in your priority list
+          - The "On Hold" filter button is now highlighted
       - step: click the "Show all" filter button
         results:
           - You are back to the original display, containing both applications and update requests
\ No newline at end of file

From 3aa6803b57bf2b8f1e32dda55d0114e79c0e8dd5 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Fri, 19 Jul 2024 13:27:38 +0100
Subject: [PATCH 39/70] remove on hold constraint from update requests (should
 be irrelevant)

---
 portality/bll/services/todo.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py
index 258094b0c..38a0225dd 100644
--- a/portality/bll/services/todo.py
+++ b/portality/bll/services/todo.py
@@ -274,8 +274,7 @@ def maned_last_month_update_requests(cls, size, maned_of):
             must_nots=[
                 TodoQuery.status([
                     constants.APPLICATION_STATUS_ACCEPTED,
-                    constants.APPLICATION_STATUS_REJECTED,
-                    constants.APPLICATION_STATUS_ON_HOLD
+                    constants.APPLICATION_STATUS_REJECTED
                 ])
                 # TodoQuery.exists("admin.editor")
             ],
@@ -298,8 +297,7 @@ def maned_new_update_requests(cls, size, maned_of):
             must_nots=[
                 TodoQuery.status([
                     constants.APPLICATION_STATUS_ACCEPTED,
-                    constants.APPLICATION_STATUS_REJECTED,
-                    constants.APPLICATION_STATUS_ON_HOLD
+                    constants.APPLICATION_STATUS_REJECTED
                 ])
                 # TodoQuery.exists("admin.editor")
             ],

From ad0c19c54b0ab230997c2158ca6c9e212453a8c6 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Tue, 30 Jul 2024 15:10:11 +0100
Subject: [PATCH 40/70] Reverted submodule edges to develop_edges1

---
 portality/static/vendor/edges | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/portality/static/vendor/edges b/portality/static/vendor/edges
index 9639b871a..2ca0da93e 160000
--- a/portality/static/vendor/edges
+++ b/portality/static/vendor/edges
@@ -1 +1 @@
-Subproject commit 9639b871acb1f6590ee78e236f2c9333479c9fe8
+Subproject commit 2ca0da93e7bf345a7e529d1ed056c8dd0a328b6a

From c12c2ab5366d4bf09c2e3a88d571dbf50fe9f27f Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 31 Jul 2024 10:09:38 +0100
Subject: [PATCH 41/70] change edges HEAD to same as develop
 990f4220163a3e18880f0bdc3ad5c80d234d22dd

---
 portality/static/vendor/edges | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/portality/static/vendor/edges b/portality/static/vendor/edges
index 2ca0da93e..990f42201 160000
--- a/portality/static/vendor/edges
+++ b/portality/static/vendor/edges
@@ -1 +1 @@
-Subproject commit 2ca0da93e7bf345a7e529d1ed056c8dd0a328b6a
+Subproject commit 990f4220163a3e18880f0bdc3ad5c80d234d22dd

From b7052394c1f3a7a1280a8dcc8ad0ac75b511a9b9 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Thu, 22 Aug 2024 16:33:53 +0100
Subject: [PATCH 42/70] add withdrawn journals as delete records in oai

---
 portality/crosswalks/oaipmh.py |  7 +++++++
 portality/models/oaipmh.py     | 15 ++++++++-------
 portality/view/oaipmh.py       | 12 ++++++++----
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/portality/crosswalks/oaipmh.py b/portality/crosswalks/oaipmh.py
index 26bab8229..a4c514988 100644
--- a/portality/crosswalks/oaipmh.py
+++ b/portality/crosswalks/oaipmh.py
@@ -246,6 +246,10 @@ def crosswalk(self, record):
             idel = etree.SubElement(oai_dc, self.DC + "identifier")
             set_text(idel, identifier.get("id"))
 
+        # beyond this point, only include the metadata if the record is not "deleted"
+        if not record.is_in_doaj():
+            return metadata
+
         # our internal identifier
         url = app.config["BASE_URL"] + "/toc/" + record.toc_id
         idel = etree.SubElement(oai_dc, self.DC + "identifier")
@@ -293,6 +297,9 @@ def header(self, record):
         bibjson = record.bibjson()
         head = etree.Element(self.PMH + "header", nsmap=self.NSMAP)
 
+        if not record.is_in_doaj():
+            head.set("status", "deleted")
+
         identifier = etree.SubElement(head, self.PMH + "identifier")
         set_text(identifier, make_oai_identifier(record.id, "journal"))
 
diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py
index 4350cb66e..39a6ee4f2 100644
--- a/portality/models/oaipmh.py
+++ b/portality/models/oaipmh.py
@@ -42,7 +42,7 @@ class OAIPMHRecord(object):
         "query": {
             "bool": {
                 "must": [
-                    { "term": { "admin.in_doaj": True } }
+                    # { "term": { "admin.in_doaj": True } }
                 ]
             }
         },
@@ -126,15 +126,16 @@ def pull(self, identifier):
             return record
         return None
 
+
 class OAIPMHJournal(OAIPMHRecord, Journal):
     def list_records(self, from_date=None, until_date=None, oai_set=None, list_size=None, start_after=None):
         total, results = super(OAIPMHJournal, self).list_records(from_date=from_date,
             until_date=until_date, oai_set=oai_set, list_size=list_size, start_after=start_after)
         return total, [Journal(**r) for r in results]
 
-    def pull(self, identifier):
-        # override the default pull, as we care about whether the item is in_doaj
-        record = super(OAIPMHJournal, self).pull(identifier)
-        if record is not None and record.is_in_doaj():
-            return record
-        return None
+    # def pull(self, identifier):
+    #     # override the default pull, as we care about whether the item is in_doaj
+    #     record = super(OAIPMHJournal, self).pull(identifier)
+    #     if record is not None and record.is_in_doaj():
+    #         return record
+    #     return None
diff --git a/portality/view/oaipmh.py b/portality/view/oaipmh.py
index 947d5d5f2..3575c4cf1 100644
--- a/portality/view/oaipmh.py
+++ b/portality/view/oaipmh.py
@@ -288,13 +288,16 @@ def get_record(dao, base_url, specified_oai_endpoint, identifier=None, metadata_
                 return IdDoesNotExist(base_url)
             # do the crosswalk
             xwalk = get_crosswalk(f.get("metadataPrefix"), dao.__type__)
-            metadata = xwalk.crosswalk(record)
+
             header = xwalk.header(record)
-            # make the response
             oai_id = make_oai_identifier(identifier, dao.__type__)
             gr = GetRecord(base_url, oai_id, metadata_prefix)
-            gr.metadata = metadata
             gr.header = header
+
+            if record.is_in_doaj():
+                metadata = xwalk.crosswalk(record)
+                gr.metadata = metadata
+
             return gr
 
     # if we have not returned already, this means we can't disseminate this format
@@ -556,7 +559,8 @@ def get_element(self):
         record = etree.SubElement(gr, self.PMH + "record")
 
         record.append(self.header)
-        record.append(self.metadata)
+        if self.metadata is not None:
+            record.append(self.metadata)
 
         return gr
 

From c0eef4e6ad0d3879ead2bc9bba4cf3a4b01057c5 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Tue, 27 Aug 2024 10:37:56 +0100
Subject: [PATCH 43/70] add withdrawn articles as delete records in oai

---
 portality/crosswalks/oaipmh.py | 82 +++++++++++++++++++++-------------
 portality/models/oaipmh.py     | 12 ++---
 2 files changed, 58 insertions(+), 36 deletions(-)

diff --git a/portality/crosswalks/oaipmh.py b/portality/crosswalks/oaipmh.py
index a4c514988..6c844a3c3 100644
--- a/portality/crosswalks/oaipmh.py
+++ b/portality/crosswalks/oaipmh.py
@@ -106,15 +106,28 @@ def crosswalk(self, record):
         oai_dc.set(self.XSI + "schemaLocation",
                    "http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd")
 
-        if bibjson.title is not None:
-            title = etree.SubElement(oai_dc, self.DC + "title")
-            set_text(title, bibjson.title)
-
         # all the external identifiers (ISSNs, etc)
         for identifier in bibjson.get_identifiers():
             idel = etree.SubElement(oai_dc, self.DC + "identifier")
             set_text(idel, identifier.get("id"))
 
+        # beyond this point, only include the metadata if the record is not "deleted"
+        if not record.is_in_doaj():
+            # include the ft url only, so the client can identify the article this way if needed
+            ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False)
+            if ftobj and "url" in ftobj:
+                urlel = etree.SubElement(oai_dc, self.DC + "relation")
+                set_text(urlel, ftobj.get("url"))
+            return metadata
+
+        for identifier in bibjson.get_identifiers(idtype=bibjson.P_ISSN) + bibjson.get_identifiers(idtype=bibjson.E_ISSN):
+            journallink = etree.SubElement(oai_dc, self.DC + "relation")
+            set_text(journallink, app.config['BASE_URL'] + "/toc/" + identifier)
+
+        if bibjson.title is not None:
+            title = etree.SubElement(oai_dc, self.DC + "title")
+            set_text(title, bibjson.title)
+
         # our internal identifier
         url = app.config['BASE_URL'] + "/article/" + record.id
         idel = etree.SubElement(oai_dc, self.DC + "identifier")
@@ -130,10 +143,6 @@ def crosswalk(self, record):
             urlel = etree.SubElement(oai_dc, self.DC + "relation")
             set_text(urlel, url.get("url"))
 
-        for identifier in bibjson.get_identifiers(idtype=bibjson.P_ISSN) + bibjson.get_identifiers(idtype=bibjson.E_ISSN):
-            journallink = etree.SubElement(oai_dc, self.DC + "relation")
-            set_text(journallink, app.config['BASE_URL'] + "/toc/" + identifier)
-
         if bibjson.abstract is not None:
             abstract = etree.SubElement(oai_dc, self.DC + "description")
             set_text(abstract, bibjson.abstract)
@@ -171,6 +180,9 @@ def header(self, record):
         bibjson = record.bibjson()
         head = etree.Element(self.PMH + "header", nsmap=self.NSMAP)
 
+        if not record.is_in_doaj():
+            head.set("status", "deleted")
+
         identifier = etree.SubElement(head, self.PMH + "identifier")
         set_text(identifier, make_oai_identifier(record.id, "article"))
 
@@ -329,6 +341,34 @@ def crosswalk(self, record):
         oai_doaj_article.set(self.XSI + "schemaLocation",
             "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd http://doaj.org/features/oai_doaj/1.0/ https://doaj.org/static/doaj/doajArticles.xsd")
 
+        # all the external identifiers (ISSNs, etc)
+        if bibjson.get_one_identifier(bibjson.P_ISSN):
+            issn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "issn")
+            set_text(issn, bibjson.get_one_identifier(bibjson.P_ISSN))
+
+        if bibjson.get_one_identifier(bibjson.E_ISSN):
+            eissn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "eissn")
+            set_text(eissn, bibjson.get_one_identifier(bibjson.E_ISSN))
+
+        if bibjson.get_one_identifier(bibjson.DOI):
+            doi = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "doi")
+            set_text(doi, bibjson.get_one_identifier(bibjson.DOI))
+
+        ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False)
+        if ftobj:
+            attrib = {}
+            if "content_type" in ftobj:
+                attrib['format'] = ftobj['content_type']
+
+            fulltext_url_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "fullTextUrl", **attrib)
+
+            if "url" in ftobj:
+                set_text(fulltext_url_elem, ftobj['url'])
+
+        # beyond this point, only include the metadata if the record is not "deleted"
+        if not record.is_in_doaj():
+            return metadata
+
         # look up the journal's language
         jlangs = bibjson.journal_language
         # first, if there are any languages recorded, get the 3-char code
@@ -355,14 +395,6 @@ def crosswalk(self, record):
             journtitel = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "journalTitle")
             set_text(journtitel, bibjson.journal_title)
 
-        # all the external identifiers (ISSNs, etc)
-        if bibjson.get_one_identifier(bibjson.P_ISSN):
-            issn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "issn")
-            set_text(issn, bibjson.get_one_identifier(bibjson.P_ISSN))
-
-        if bibjson.get_one_identifier(bibjson.E_ISSN):
-            eissn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "eissn")
-            set_text(eissn, bibjson.get_one_identifier(bibjson.E_ISSN))
 
         # work out the date of publication
         date = bibjson.get_publication_date()
@@ -396,9 +428,7 @@ def crosswalk(self, record):
             end_page = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "endPage")
             set_text(end_page, bibjson.end_page)
 
-        if bibjson.get_one_identifier(bibjson.DOI):
-            doi = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "doi")
-            set_text(doi, bibjson.get_one_identifier(bibjson.DOI))
+
 
         if record.publisher_record_id():
             pubrecid = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "publisherRecordId")
@@ -443,17 +473,6 @@ def crosswalk(self, record):
             abstract = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "abstract")
             set_text(abstract, bibjson.abstract)
 
-        ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False)
-        if ftobj:
-            attrib = {}
-            if "content_type" in ftobj:
-                attrib['format'] = ftobj['content_type']
-
-            fulltext_url_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "fullTextUrl", **attrib)
-
-            if "url" in ftobj:
-                set_text(fulltext_url_elem, ftobj['url'])
-
         if bibjson.keywords:
             keywords_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + 'keywords')
             for keyword in bibjson.keywords:
@@ -466,6 +485,9 @@ def header(self, record):
         bibjson = record.bibjson()
         head = etree.Element(self.PMH + "header", nsmap=self.NSMAP)
 
+        if not record.is_in_doaj():
+            head.set("status", "deleted")
+
         identifier = etree.SubElement(head, self.PMH + "identifier")
         set_text(identifier, make_oai_identifier(record.id, "article"))
 
diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py
index 39a6ee4f2..461cd9759 100644
--- a/portality/models/oaipmh.py
+++ b/portality/models/oaipmh.py
@@ -119,12 +119,12 @@ def list_records(self, from_date=None, until_date=None, oai_set=None, list_size=
             until_date=until_date, oai_set=oai_set, list_size=list_size, start_after=start_after)
         return total, [Article(**r) for r in results]
 
-    def pull(self, identifier):
-        # override the default pull, as we care about whether the item is in_doaj
-        record = super(OAIPMHArticle, self).pull(identifier)
-        if record is not None and record.is_in_doaj():
-            return record
-        return None
+    # def pull(self, identifier):
+    #     # override the default pull, as we care about whether the item is in_doaj
+    #     record = super(OAIPMHArticle, self).pull(identifier)
+    #     if record is not None and record.is_in_doaj():
+    #         return record
+    #     return None
 
 
 class OAIPMHJournal(OAIPMHRecord, Journal):

From 6485715ac1e50f6bda62571e44e44c910cdeda3e Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Wed, 28 Aug 2024 10:03:57 +0100
Subject: [PATCH 44/70] handle NoSuchObjectException (DOAJ/doajPM#3838)

---
 doajtest/unit/test_view_publisher.py | 21 +++++++++++++++++++++
 portality/view/publisher.py          |  8 ++++++--
 2 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 doajtest/unit/test_view_publisher.py

diff --git a/doajtest/unit/test_view_publisher.py b/doajtest/unit/test_view_publisher.py
new file mode 100644
index 000000000..6ea8f1ec2
--- /dev/null
+++ b/doajtest/unit/test_view_publisher.py
@@ -0,0 +1,21 @@
+from doajtest import helpers
+from doajtest.helpers import DoajTestCase
+from portality import models, constants
+from portality.util import url_for
+
+
+class TestViewPublisher(DoajTestCase):
+
+    def test_delete_application__no_such_object(self):
+        pwd = 'password'
+        un = 'publisher_a'
+        acc = models.Account.make_account(un + "@example.com", un, "Publisher " + un, [constants.ROLE_PUBLISHER])
+        acc.set_password(pwd)
+        acc.save(blocking=True)
+
+        with self.app_test.test_client() as t_client:
+            resp = helpers.login(t_client, acc.email, pwd)
+            assert resp.status_code == 200
+
+            resp = t_client.get(url_for("publisher.delete_application", application_id='no_such_id'))
+            assert resp.status_code == 400
diff --git a/portality/view/publisher.py b/portality/view/publisher.py
index a3e328b28..fe92f542a 100644
--- a/portality/view/publisher.py
+++ b/portality/view/publisher.py
@@ -4,7 +4,8 @@
 
 from portality.app_email import EmailException
 from portality import models, constants
-from portality.bll.exceptions import AuthoriseException, ArticleMergeConflict, DuplicateArticleException, ArticleNotAcceptable
+from portality.bll.exceptions import AuthoriseException, ArticleMergeConflict, DuplicateArticleException, \
+    ArticleNotAcceptable, NoSuchObjectException
 from portality.decorators import ssl_required, restrict_to_role, write_required
 from portality.dao import ESMappingMissingError
 from portality.forms.application_forms import ApplicationFormFactory
@@ -54,7 +55,10 @@ def delete_application(application_id):
 
     # otherwise delegate to the application service to sort this out
     appService = DOAJ.applicationService()
-    appService.delete_application(application_id, current_user._get_current_object())
+    try:
+        appService.delete_application(application_id, current_user._get_current_object())
+    except NoSuchObjectException:
+        abort(400)
 
     return redirect(url_for("publisher.deleted_thanks"))
 

From 9f0540315971e03e6f76f50ef788df598c0edce5 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Thu, 29 Aug 2024 11:52:16 +0100
Subject: [PATCH 45/70] tighten up deleted record implementation for spec
 compliance

---
 portality/crosswalks/oaipmh.py | 89 +++++++++++++++-------------------
 portality/view/oaipmh.py       |  3 +-
 2 files changed, 41 insertions(+), 51 deletions(-)

diff --git a/portality/crosswalks/oaipmh.py b/portality/crosswalks/oaipmh.py
index 6c844a3c3..cd09e6089 100644
--- a/portality/crosswalks/oaipmh.py
+++ b/portality/crosswalks/oaipmh.py
@@ -99,6 +99,9 @@ class OAI_DC_Article(OAI_DC):
     ~~->OAIDC:Crosswalk~~
     """
     def crosswalk(self, record):
+        if not record.is_in_doaj():
+            return None
+
         bibjson = record.bibjson()
 
         metadata = etree.Element(self.PMH + "metadata")
@@ -106,28 +109,15 @@ def crosswalk(self, record):
         oai_dc.set(self.XSI + "schemaLocation",
                    "http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd")
 
+        if bibjson.title is not None:
+            title = etree.SubElement(oai_dc, self.DC + "title")
+            set_text(title, bibjson.title)
+
         # all the external identifiers (ISSNs, etc)
         for identifier in bibjson.get_identifiers():
             idel = etree.SubElement(oai_dc, self.DC + "identifier")
             set_text(idel, identifier.get("id"))
 
-        # beyond this point, only include the metadata if the record is not "deleted"
-        if not record.is_in_doaj():
-            # include the ft url only, so the client can identify the article this way if needed
-            ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False)
-            if ftobj and "url" in ftobj:
-                urlel = etree.SubElement(oai_dc, self.DC + "relation")
-                set_text(urlel, ftobj.get("url"))
-            return metadata
-
-        for identifier in bibjson.get_identifiers(idtype=bibjson.P_ISSN) + bibjson.get_identifiers(idtype=bibjson.E_ISSN):
-            journallink = etree.SubElement(oai_dc, self.DC + "relation")
-            set_text(journallink, app.config['BASE_URL'] + "/toc/" + identifier)
-
-        if bibjson.title is not None:
-            title = etree.SubElement(oai_dc, self.DC + "title")
-            set_text(title, bibjson.title)
-
         # our internal identifier
         url = app.config['BASE_URL'] + "/article/" + record.id
         idel = etree.SubElement(oai_dc, self.DC + "identifier")
@@ -143,6 +133,10 @@ def crosswalk(self, record):
             urlel = etree.SubElement(oai_dc, self.DC + "relation")
             set_text(urlel, url.get("url"))
 
+        for identifier in bibjson.get_identifiers(idtype=bibjson.P_ISSN) + bibjson.get_identifiers(idtype=bibjson.E_ISSN):
+            journallink = etree.SubElement(oai_dc, self.DC + "relation")
+            set_text(journallink, app.config['BASE_URL'] + "/toc/" + identifier)
+
         if bibjson.abstract is not None:
             abstract = etree.SubElement(oai_dc, self.DC + "description")
             set_text(abstract, bibjson.abstract)
@@ -243,6 +237,9 @@ class OAI_DC_Journal(OAI_DC):
     ~~->OAIDC:Crosswalk~~
     """
     def crosswalk(self, record):
+        if not record.is_in_doaj():
+            return None
+
         bibjson = record.bibjson()
 
         metadata = etree.Element(self.PMH + "metadata")
@@ -258,10 +255,6 @@ def crosswalk(self, record):
             idel = etree.SubElement(oai_dc, self.DC + "identifier")
             set_text(idel, identifier.get("id"))
 
-        # beyond this point, only include the metadata if the record is not "deleted"
-        if not record.is_in_doaj():
-            return metadata
-
         # our internal identifier
         url = app.config["BASE_URL"] + "/toc/" + record.toc_id
         idel = etree.SubElement(oai_dc, self.DC + "identifier")
@@ -334,6 +327,9 @@ class OAI_DOAJ_Article(OAI_Crosswalk):
     NSMAP.update({"oai_doaj": OAI_DOAJ_NAMESPACE})
 
     def crosswalk(self, record):
+        if not record.is_in_doaj():
+            return None
+
         bibjson = record.bibjson()
 
         metadata = etree.Element(self.PMH + "metadata")
@@ -341,34 +337,6 @@ def crosswalk(self, record):
         oai_doaj_article.set(self.XSI + "schemaLocation",
             "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd http://doaj.org/features/oai_doaj/1.0/ https://doaj.org/static/doaj/doajArticles.xsd")
 
-        # all the external identifiers (ISSNs, etc)
-        if bibjson.get_one_identifier(bibjson.P_ISSN):
-            issn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "issn")
-            set_text(issn, bibjson.get_one_identifier(bibjson.P_ISSN))
-
-        if bibjson.get_one_identifier(bibjson.E_ISSN):
-            eissn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "eissn")
-            set_text(eissn, bibjson.get_one_identifier(bibjson.E_ISSN))
-
-        if bibjson.get_one_identifier(bibjson.DOI):
-            doi = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "doi")
-            set_text(doi, bibjson.get_one_identifier(bibjson.DOI))
-
-        ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False)
-        if ftobj:
-            attrib = {}
-            if "content_type" in ftobj:
-                attrib['format'] = ftobj['content_type']
-
-            fulltext_url_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "fullTextUrl", **attrib)
-
-            if "url" in ftobj:
-                set_text(fulltext_url_elem, ftobj['url'])
-
-        # beyond this point, only include the metadata if the record is not "deleted"
-        if not record.is_in_doaj():
-            return metadata
-
         # look up the journal's language
         jlangs = bibjson.journal_language
         # first, if there are any languages recorded, get the 3-char code
@@ -395,6 +363,14 @@ def crosswalk(self, record):
             journtitel = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "journalTitle")
             set_text(journtitel, bibjson.journal_title)
 
+        # all the external identifiers (ISSNs, etc)
+        if bibjson.get_one_identifier(bibjson.P_ISSN):
+            issn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "issn")
+            set_text(issn, bibjson.get_one_identifier(bibjson.P_ISSN))
+
+        if bibjson.get_one_identifier(bibjson.E_ISSN):
+            eissn = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "eissn")
+            set_text(eissn, bibjson.get_one_identifier(bibjson.E_ISSN))
 
         # work out the date of publication
         date = bibjson.get_publication_date()
@@ -428,7 +404,9 @@ def crosswalk(self, record):
             end_page = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "endPage")
             set_text(end_page, bibjson.end_page)
 
-
+        if bibjson.get_one_identifier(bibjson.DOI):
+            doi = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "doi")
+            set_text(doi, bibjson.get_one_identifier(bibjson.DOI))
 
         if record.publisher_record_id():
             pubrecid = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "publisherRecordId")
@@ -473,6 +451,17 @@ def crosswalk(self, record):
             abstract = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "abstract")
             set_text(abstract, bibjson.abstract)
 
+        ftobj = bibjson.get_single_url('fulltext', unpack_urlobj=False)
+        if ftobj:
+            attrib = {}
+            if "content_type" in ftobj:
+                attrib['format'] = ftobj['content_type']
+
+            fulltext_url_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + "fullTextUrl", **attrib)
+
+            if "url" in ftobj:
+                set_text(fulltext_url_elem, ftobj['url'])
+
         if bibjson.keywords:
             keywords_elem = etree.SubElement(oai_doaj_article, self.OAI_DOAJ + 'keywords')
             for keyword in bibjson.keywords:
diff --git a/portality/view/oaipmh.py b/portality/view/oaipmh.py
index 3575c4cf1..b1c5ec7ac 100644
--- a/portality/view/oaipmh.py
+++ b/portality/view/oaipmh.py
@@ -739,7 +739,8 @@ def get_element(self):
         for metadata, header in self.records:
             r = etree.SubElement(lr, self.PMH + "record")
             r.append(header)
-            r.append(metadata)
+            if metadata is not None:
+                r.append(metadata)
 
         if self.resumption is not None:
             rt = etree.SubElement(lr, self.PMH + "resumptionToken")

From 0aaa3db0f83b3e64eee34efaf520a38dfb0b5625 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Thu, 29 Aug 2024 13:11:16 +0100
Subject: [PATCH 46/70] Update oai tests

---
 doajtest/unit/test_oaipmh.py | 71 +++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 21 deletions(-)

diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py
index 0a6540eec..a12f9736b 100644
--- a/doajtest/unit/test_oaipmh.py
+++ b/doajtest/unit/test_oaipmh.py
@@ -43,7 +43,7 @@ def test_01_oai_ListMetadataFormats(self):
                 assert t.xpath('/oai:OAI-PMH/oai:ListMetadataFormats/oai:metadataFormat/oai:metadataPrefix', namespaces=self.oai_ns)[0].text == 'oai_dc'
 
     def test_02_oai_journals(self):
-        """test if the OAI-PMH journal feed returns records and only displays journals accepted in DOAJ"""
+        """test if the OAI-PMH journal feed returns records and only displays journals accepted in DOAJ, marking withdrawn ones as deleted"""
         journal_sources = JournalFixtureFactory.make_many_journal_sources(2, in_doaj=True)
         j_public = models.Journal(**journal_sources[0])
         j_public.save(blocking=True)
@@ -52,6 +52,7 @@ def test_02_oai_journals(self):
         j_private = models.Journal(**journal_sources[1])
         j_private.set_in_doaj(False)
         j_private.save(blocking=True)
+        deleted_id = j_private.id
 
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
@@ -61,11 +62,24 @@ def test_02_oai_journals(self):
                 t = etree.fromstring(resp.data)
                 records = t.xpath('/oai:OAI-PMH/oai:ListRecords', namespaces=self.oai_ns)
 
-                # Check we only have one journal returned
-                assert len(records[0].xpath('//oai:record', namespaces=self.oai_ns)) == 1
-
-                # Check we have the correct journal
-                assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text == j_public.bibjson().title
+                # Check we only have two journals returned
+                assert len(records[0].xpath('//oai:record', namespaces=self.oai_ns)) == 2
+
+                seen_deleted = False
+                seen_public = False
+                records = records[0].getchildren()
+                for r in records:
+                    header = r.xpath('oai:header', namespaces=self.oai_ns)[0]
+                    status = header.get("status")
+                    if status == "deleted":
+                        seen_deleted = True
+                    else:
+                        # Check we have the correct journal
+                        seen_public = True
+                        assert r.xpath('//dc:title', namespaces=self.oai_ns)[0].text == j_public.bibjson().title
+
+                assert seen_deleted
+                assert seen_public
 
                 resp = t_client.get(url_for('oaipmh.oaipmh', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier={0}'.format(public_id))
                 assert resp.status_code == 200
@@ -306,23 +320,22 @@ def test_08_list_sets(self):
 
 
     def test_09_article(self):
-        """test if the OAI-PMH journal feed returns records and only displays journals accepted in DOAJ"""
+        """test if the OAI-PMH article feed returns records and only displays articles accepted in DOAJ, showing the others as deleted"""
         article_source = ArticleFixtureFactory.make_article_source(eissn='1234-1234', pissn='5678-5678,', in_doaj=False)
-        """test if the OAI-PMH article feed returns records and only displays articles accepted in DOAJ"""
         a_private = models.Article(**article_source)
+        a_private.set_id(a_private.makeid())
         ba = a_private.bibjson()
         ba.title = "Private Article"
         a_private.save(blocking=True)
 
         article_source = ArticleFixtureFactory.make_article_source(eissn='4321-4321', pissn='8765-8765,', in_doaj=True)
         a_public = models.Article(**article_source)
+        a_public.set_id(a_public.makeid())
         ba = a_public.bibjson()
         ba.title = "Public Article"
         a_public.save(blocking=True)
         public_id = a_public.id
 
-        time.sleep(1)
-
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
                 resp = t_client.get(url_for('oaipmh.oaipmh',  specified='article', verb='ListRecords', metadataPrefix='oai_dc'))
@@ -331,23 +344,39 @@ def test_09_article(self):
                 t = etree.fromstring(resp.data)
                 records = t.xpath('/oai:OAI-PMH/oai:ListRecords', namespaces=self.oai_ns)
 
-                # Check we only have one journal returned
+                # Check we only have two articles returned
                 r = records[0].xpath('//oai:record', namespaces=self.oai_ns)
-                assert len(r) == 1
-
-                # Check we have the correct journal
-                title = r[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text
-                # check orcid_id xwalk
-                assert str(records[0].xpath('//dc:creator/@id', namespaces=self.oai_ns)[0]) == a_public.bibjson().author[0].get("orcid_id")
-                assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text == a_public.bibjson().title
-
-                resp = t_client.get(url_for('oaipmh.oaipmh',  specified='article', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier=abcdefghijk_article')
+                assert len(r) == 2
+
+                seen_deleted = False
+                seen_public = False
+                records = records[0].getchildren()
+                for r in records:
+                    header = r.xpath('oai:header', namespaces=self.oai_ns)[0]
+                    status = header.get("status")
+                    if status == "deleted":
+                        seen_deleted = True
+                    else:
+                        seen_public = True
+                        # Check we have the correct article
+                        title = r[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text
+
+                        # check orcid_id xwalk
+                        assert str(records[0].xpath('//dc:creator/@id', namespaces=self.oai_ns)[0]) == \
+                               a_public.bibjson().author[0].get("orcid_id")
+                        assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[
+                                   0].text == a_public.bibjson().title
+
+                assert seen_deleted
+                assert seen_public
+
+                resp = t_client.get(url_for('oaipmh.oaipmh',  specified='article', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier=' + public_id)
                 assert resp.status_code == 200
 
                 t = etree.fromstring(resp.data)
                 records = t.xpath('/oai:OAI-PMH/oai:GetRecord', namespaces=self.oai_ns)
 
-                # Check we only have one journal returnedt
+                # Check we only have one article returned
                 kids = records[0].getchildren()
                 r = records[0].xpath('//oai:record', namespaces=self.oai_ns)
                 assert len(r) == 1

From e4f03ffc3fad1aadacc57ce2684fc946b534468e Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Thu, 29 Aug 2024 13:25:25 +0100
Subject: [PATCH 47/70] 404

---
 doajtest/unit/test_view_publisher.py | 2 +-
 portality/view/publisher.py          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doajtest/unit/test_view_publisher.py b/doajtest/unit/test_view_publisher.py
index 6ea8f1ec2..fbdf77054 100644
--- a/doajtest/unit/test_view_publisher.py
+++ b/doajtest/unit/test_view_publisher.py
@@ -18,4 +18,4 @@ def test_delete_application__no_such_object(self):
             assert resp.status_code == 200
 
             resp = t_client.get(url_for("publisher.delete_application", application_id='no_such_id'))
-            assert resp.status_code == 400
+            assert resp.status_code == 404
diff --git a/portality/view/publisher.py b/portality/view/publisher.py
index fe92f542a..902be3960 100644
--- a/portality/view/publisher.py
+++ b/portality/view/publisher.py
@@ -58,7 +58,7 @@ def delete_application(application_id):
     try:
         appService.delete_application(application_id, current_user._get_current_object())
     except NoSuchObjectException:
-        abort(400)
+        abort(404)
 
     return redirect(url_for("publisher.deleted_thanks"))
 

From d6b2565821559b2b36c5d98509089c96deaddaff Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Thu, 29 Aug 2024 13:33:08 +0100
Subject: [PATCH 48/70] timing fix to oai test

---
 doajtest/unit/test_oaipmh.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py
index a12f9736b..d1f6133b4 100644
--- a/doajtest/unit/test_oaipmh.py
+++ b/doajtest/unit/test_oaipmh.py
@@ -336,6 +336,8 @@ def test_09_article(self):
         a_public.save(blocking=True)
         public_id = a_public.id
 
+        time.sleep(1)
+
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
                 resp = t_client.get(url_for('oaipmh.oaipmh',  specified='article', verb='ListRecords', metadataPrefix='oai_dc'))

From 8956178f8703b7a4a4746f2a6d0c72110bb63470 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Tue, 3 Sep 2024 15:47:46 +0100
Subject: [PATCH 49/70] put new dependency in alphabetical order

---
 setup.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 3fca37d02..870d178cb 100644
--- a/setup.py
+++ b/setup.py
@@ -19,6 +19,7 @@
         "feedparser==6.0.8",
         "itsdangerous==2.0.1",     # fixme: unpinned dependency of flask, 2.1.0 is causing an import error 'json'
         "jinja2<3.1.0",            # fixme: unpinned dependency of flask, import error on 'escape'
+        "jsonpath-ng~=1.6",
         "Flask~=2.1.2",
         "Flask-Cors==3.0.8",
         "Flask-DebugToolbar==0.13.1",
@@ -64,8 +65,6 @@
         'gspread-dataframe~=3.3.1',
         'gspread-formatting~=1.1.2',
 
-        'jsonpath-ng~=1.6',
-
     ] + (["setproctitle==1.1.10"] if "linux" in sys.platform else []),
     extras_require={
         # prevent backtracking through all versions

From d8974c9b3273315af524b1aa6a96fc129a348857 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Tue, 3 Sep 2024 16:53:06 +0100
Subject: [PATCH 50/70] update on hold dashboard rule to cover maned or
 assigned user

---
 doajtest/testbook/dashboards/maned_todo.yml   | 18 +++++++------
 .../testdrive/todo_maned_editor_associate.py  | 16 +++++++++---
 portality/bll/services/todo.py                | 26 +++++++++++++------
 3 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/doajtest/testbook/dashboards/maned_todo.yml b/doajtest/testbook/dashboards/maned_todo.yml
index 017965bc7..06ce8921c 100644
--- a/doajtest/testbook/dashboards/maned_todo.yml
+++ b/doajtest/testbook/dashboards/maned_todo.yml
@@ -27,7 +27,7 @@ tests:
       - step: Go to the maned dashboard page
         path: /dashboard
         results:
-          - You can see 17 applications in your priority list
+          - You can see 18 applications in your priority list
           - Your priority list contains a mixture of managing editor items (actions related to teams you are the managing editor for), 
             editor items (actions related to teams you are the editor for) and associate items (actions related to applications which
             are assigned specifically to you for review).
@@ -43,25 +43,25 @@ tests:
       - step: Change the application status to "Accepted" and save
       - step: close the tab, return to the dashboard and reload the page
         results:
-          - You can see 16 applications in your priority list
+          - You can see 17 applications in your priority list
           - The application you have just edited has disappeared from your priority list
       - step: click on the [in progress] stalled managing editor's application
       - step: make any minor adjustment to the metadata and save
       - step: close the tab, return to the dashboard and reload the page
         results:
-          - You can see 15 applications in your priority list
+          - You can see 16 applications in your priority list
           - The application you just edited has disappeared from your priority list
       - step: click on the "completed" maned application
       - step: Change the application to "ready" status
       - step: close the tab, return to the dashboard and reload the page
         results:
-          - You can still see 13 applications in your priority list
+          - You can still see 15 applications in your priority list
           - The completed application you just moved to ready is now in your priority list as a ready application
       - step: click on the pending managing editor's application
       - step: Assign the item to an editor in the selected group (there should be a test editor available to you to select)
       - step: close the tab, return to the dashboard and reload the page
         results:
-          - You have 12 applications left in your todo list
+          - You have 14 applications left in your todo list
           - The pending application you just edited is no longer visible
 
   - title: Filtering the todo list
@@ -75,14 +75,14 @@ tests:
       - step: Go to the maned dashboard page
         path: /dashboard
         results:
-          - You can see 17 applications in your priority list
+          - You can see 18 applications in your priority list
           - Your highest priority item (first in the list) is for an update request which was submitted last month
           - Your lowest priority item (last in the list) is for an update request which was submitted this month
           - On the top right of the todo list are a set of filter buttons "Show all", "New Applications", "Update Requests" and "On Hold"
           - The "Show all" button is highlighted
       - step: click on the "New Applications" filter button
         results:
-          - You can see 14 applications in your priority list
+          - You can see 16 applications in your priority list
           - The update requests and "on hold" items which were on the previous screen are no longer visible
           - The "New Applications" filter button is now highlighted
       - step: click on the "Update Request" filter button
@@ -93,8 +93,10 @@ tests:
           - The "Update Request" filter button is now highlighted
       - step: click on the "On Hold" filter button
         results:
-          - You can see 1 application in your priority list
+          - You can see 2 application in your priority list
           - The "On Hold" filter button is now highlighted
+          - One of the "On Hold" items is for an application which is not assigned to you, but belongs to a group you are the managing editor for
+          - The other "On Hold" item is for an application which is assigned to you, in a group for which you are not the managing editor
       - step: click the "Show all" filter button
         results:
           - You are back to the original display, containing both applications and update requests
\ No newline at end of file
diff --git a/doajtest/testdrive/todo_maned_editor_associate.py b/doajtest/testdrive/todo_maned_editor_associate.py
index 0ca722c55..f6a4d6287 100644
--- a/doajtest/testdrive/todo_maned_editor_associate.py
+++ b/doajtest/testdrive/todo_maned_editor_associate.py
@@ -51,7 +51,7 @@ def setup(self) -> dict:
 
         aapps = build_associate_applications(un)
         eapps = build_editor_applications(un, eg2)
-        mapps = build_maned_applications(un, eg1, owner.id, eg3)
+        mapps = build_maned_applications(un, eg1, owner.id, eg3, eg2)
 
 
         return {
@@ -96,7 +96,7 @@ def teardown(self, params) -> dict:
         return {"status": "success"}
 
 
-def build_maned_applications(un, eg, owner, eponymous_group):
+def build_maned_applications(un, eg, owner, eponymous_group, other_group):
     w = 7 * 24 * 60 * 60
 
     apps = {}
@@ -142,14 +142,22 @@ def build_maned_applications(un, eg, owner, eponymous_group):
         "title": un + " Maned Pending Application"
     }]
 
-    app = build_application(un + " Maned On Hold Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD,
+    app = build_application(un + " Maned (Group) On Hold Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD,
                             editor_group=eg.name, owner=owner)
     app.save()
     apps["on_hold"] = [{
         "id": app.id,
-        "title": un + " Maned On Hold Application"
+        "title": un + " Maned (Group) On Hold Application"
     }]
 
+    app = build_application(un + " Maned (Editor) On Hold Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_ON_HOLD,
+                            editor_group=other_group.name, editor=un, owner=owner)
+    app.save()
+    apps["on_hold"].append({
+        "id": app.id,
+        "title": un + " Maned (Editor) On Hold Application"
+    })
+
     app = build_application(un + " Maned Low Priority Pending Application", 1 * w, 1 * w,
                             constants.APPLICATION_STATUS_PENDING,
                             editor_group=eponymous_group.name, owner=owner)
diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py
index 38a0225dd..cd3723bb3 100644
--- a/portality/bll/services/todo.py
+++ b/portality/bll/services/todo.py
@@ -90,7 +90,7 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True
                 queries.append(TodoRules.maned_last_month_update_requests(size, maned_of))
                 queries.append(TodoRules.maned_new_update_requests(size, maned_of))
             if on_hold:
-                queries.append(TodoRules.maned_on_hold(size, maned_of))
+                queries.append(TodoRules.maned_on_hold(size, account.id, maned_of))
 
         if new_applications:    # editor and associate editor roles only deal with new applications
             if account.has_role("editor"):
@@ -307,14 +307,17 @@ def maned_new_update_requests(cls, size, maned_of):
         return constants.TODO_MANED_NEW_UPDATE_REQUEST, assign_pending, sort_date, -2
 
     @classmethod
-    def maned_on_hold(cls, size, maned_of):
+    def maned_on_hold(cls, size, account, maned_of):
         sort_date = "created_date"
         on_holds = TodoQuery(
             musts=[
-                TodoQuery.editor_group(maned_of),
                 TodoQuery.is_new_application(),
                 TodoQuery.status([constants.APPLICATION_STATUS_ON_HOLD])
             ],
+            ors=[
+                TodoQuery.editor_group(maned_of),
+                TodoQuery.editor(account)
+            ],
             sort=sort_date,
             size=size
         )
@@ -484,9 +487,10 @@ class TodoQuery(object):
     # therefore, we take a created_date sort to mean a date_applied sort
     cd_sort = {"admin.date_applied": {"order": "asc"}}
 
-    def __init__(self, musts=None, must_nots=None, sort="last_manual_update", size=10):
+    def __init__(self, musts=None, must_nots=None, ors=None, sort="last_manual_update", size=10):
         self._musts = [] if musts is None else musts
         self._must_nots = [] if must_nots is None else must_nots
+        self._ors = [] if ors is None else ors
         self._sort = sort
         self._size = size
 
@@ -494,16 +498,22 @@ def query(self):
         sort = self.lmu_sort if self._sort == "last_manual_update" else self.cd_sort
         q = {
             "query" : {
-                "bool" : {
-                    "must": self._musts,
-                    "must_not": self._must_nots
-                }
+                "bool" : {}
             },
             "sort" : [
                 sort
             ],
             "size" : self._size
         }
+
+        if len(self._musts) > 0:
+            q["query"]["bool"]["must"] = self._musts
+        if len(self._must_nots) > 0:
+            q["query"]["bool"]["must_not"] = self._must_nots
+        if len(self._ors) > 0:
+            q["query"]["bool"]["should"] = self._ors
+            q["query"]["bool"]["minimum_should_match"] = 1
+
         return q
 
     @classmethod

From eb7c05a42575c463e9f8e0496257c74556ce5001 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Mon, 16 Sep 2024 14:49:12 +0100
Subject: [PATCH 51/70] fully implement tests for pmh delete

---
 doajtest/helpers.py            | 28 ++++++++++++++---
 doajtest/unit/test_models.py   | 55 ++++++++++++++++++++++++++++++++++
 doajtest/unit/test_oaipmh.py   | 22 ++++++++++----
 portality/crosswalks/oaipmh.py |  6 ++--
 portality/models/__init__.py   |  2 +-
 portality/models/article.py    | 42 ++++++++++++++++++++++++--
 portality/models/oaipmh.py     | 21 +++++++------
 portality/settings.py          |  1 +
 8 files changed, 152 insertions(+), 25 deletions(-)

diff --git a/doajtest/helpers.py b/doajtest/helpers.py
index 1fcf47eba..19c4629b2 100644
--- a/doajtest/helpers.py
+++ b/doajtest/helpers.py
@@ -21,6 +21,7 @@
 from portality.lib.thread_utils import wait_until
 from portality.tasks.redis_huey import main_queue, long_running
 from portality.util import url_for
+from view.admin import index
 
 
 def patch_config(inst, properties):
@@ -67,6 +68,8 @@ def setUp(self):
         for im in self.warm_mappings:
             if im == "article":
                 self.warmArticle()
+            if im == "article_tombstone":
+                self.warmArticleTombstone()
             # add more types if they are necessary
 
     def tearDown(self):
@@ -82,6 +85,16 @@ def warmArticle(self):
         article.delete()
         Article.blockdeleted(article.id)
 
+    def warmArticleTombstone(self):
+        # push an article to initialise the mappings
+        from doajtest.fixtures import ArticleFixtureFactory
+        from portality.models import ArticleTombstone
+        source = ArticleFixtureFactory.make_article_source()
+        article = ArticleTombstone(**source)
+        article.save(blocking=True)
+        article.delete()
+        ArticleTombstone.blockdeleted(article.id)
+
 
 CREATED_INDICES = []
 
@@ -91,10 +104,17 @@ def initialise_index():
 
 
 def create_index(index_type):
-    if index_type in CREATED_INDICES:
-        return
-    core.initialise_index(app, core.es_connection, only_mappings=[index_type])
-    CREATED_INDICES.append(index_type)
+    if "," in index_type:
+        # this covers a DAO that has multiple index types for searching purposes
+        # expressed as a comma separated list
+        index_types = index_type.split(",")
+    else:
+        index_types = [index_type]
+    for it in index_types:
+        if it in CREATED_INDICES:
+            return
+        core.initialise_index(app, core.es_connection, only_mappings=[it])
+        CREATED_INDICES.append(it)
 
 
 def dao_proxy(dao_method, type="class"):
diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py
index af07a9285..d1bae191c 100644
--- a/doajtest/unit/test_models.py
+++ b/doajtest/unit/test_models.py
@@ -1722,6 +1722,61 @@ def test_40_autocheck_retrieves(self):
         ap2 = models.Autocheck.for_journal("9876")
         assert ap2.journal == "9876"
 
+    def test_41_article_tombstone(self):
+        t = models.ArticleTombstone()
+        t.set_id("1234")
+        t.bibjson().add_subject("LCC", "Medicine", "KM22")
+        t.set_in_doaj(True) # should have no effect
+
+        t.save(blocking=True)
+
+        t2 = models.ArticleTombstone.pull("1234")
+        assert t2.id == "1234"
+        assert t2.is_in_doaj() is False
+        assert t2.last_updated is not None
+        assert t2.bibjson().subjects()[0].get("scheme") == "LCC"
+        assert t2.bibjson().subjects()[0].get("term") == "Medicine"
+        assert t2.bibjson().subjects()[0].get("code") == "KM22"
+
+    def test_42_make_article_tombstone(self):
+        a = models.Article(**ArticleFixtureFactory.make_article_source(in_doaj=True))
+        a.set_id(a.makeid())
+
+        t = a._tombstone()
+        assert t.id == a.id
+        assert t.bibjson().subjects() == a.bibjson().subjects()
+        assert t.is_in_doaj() is False
+
+        a = models.Article(**ArticleFixtureFactory.make_article_source(in_doaj=True))
+        a.set_id(a.makeid())
+        a.delete()
+        time.sleep(1)
+
+        stone = models.ArticleTombstone.pull(a.id)
+        assert stone is not None
+
+        a = models.Article(**ArticleFixtureFactory.make_article_source(in_doaj=True))
+        a.set_id(a.makeid())
+        a.save(blocking=True)
+
+        query = {
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"id.exact": a.id}}
+                    ]
+                }
+            }
+        }
+        models.Article.delete_selected(query)
+        time.sleep(1)
+
+        stone = models.ArticleTombstone.pull(a.id)
+        assert stone is not None
+
+
+
+
 class TestAccount(DoajTestCase):
     def test_get_name_safe(self):
 
diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py
index d1f6133b4..55f27467c 100644
--- a/doajtest/unit/test_oaipmh.py
+++ b/doajtest/unit/test_oaipmh.py
@@ -10,12 +10,14 @@
 from doajtest.fixtures import ArticleFixtureFactory
 from doajtest.fixtures import JournalFixtureFactory
 from doajtest.helpers import DoajTestCase
+from models import ArticleTombstone
 from portality import models
 from portality.app import app
 from portality.lib import dates
 from portality.lib.dates import FMT_DATE_STD
 from portality.view.oaipmh import ResumptionTokenException, decode_resumption_token
 
+from doajtest.helpers import with_es
 
 class TestClient(DoajTestCase):
     @classmethod
@@ -336,7 +338,14 @@ def test_09_article(self):
         a_public.save(blocking=True)
         public_id = a_public.id
 
-        time.sleep(1)
+        stone = models.ArticleTombstone()
+        stone.set_id(stone.makeid())
+        stone.bibjson().add_subject("LCC", "Economic theory. Demography", "AB22")
+        stone.save(blocking=True)
+        stone_id = stone.id
+
+        models.Article.blockall([(a_private.id, a_private.last_updated), (a_public.id, a_public.last_updated)])
+        models.ArticleTombstone.blockall([(stone.id, stone.last_updated)])
 
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
@@ -348,16 +357,16 @@ def test_09_article(self):
 
                 # Check we only have two articles returned
                 r = records[0].xpath('//oai:record', namespaces=self.oai_ns)
-                assert len(r) == 2
+                assert len(r) == 3
 
-                seen_deleted = False
+                seen_deleted = 0
                 seen_public = False
                 records = records[0].getchildren()
                 for r in records:
                     header = r.xpath('oai:header', namespaces=self.oai_ns)[0]
                     status = header.get("status")
                     if status == "deleted":
-                        seen_deleted = True
+                        seen_deleted += 1
                     else:
                         seen_public = True
                         # Check we have the correct article
@@ -369,7 +378,7 @@ def test_09_article(self):
                         assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[
                                    0].text == a_public.bibjson().title
 
-                assert seen_deleted
+                assert seen_deleted == 2
                 assert seen_public
 
                 resp = t_client.get(url_for('oaipmh.oaipmh',  specified='article', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier=' + public_id)
@@ -437,7 +446,8 @@ def test_10_subjects(self):
                 # Check we have the correct journal
                 assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text == j_public.bibjson().title
 
-
+    @with_es(indices=[models.Article.__type__, models.ArticleTombstone.__type__],
+             warm_mappings=[models.Article.__type__, models.ArticleTombstone.__type__])
     def test_11_oai_dc_attr(self):
         """test if the OAI-PMH article feed returns record with correct attributes in oai_dc element"""
         article_source = ArticleFixtureFactory.make_article_source(eissn='1234-1234', pissn='5678-5678,', in_doaj=True)
diff --git a/portality/crosswalks/oaipmh.py b/portality/crosswalks/oaipmh.py
index cd09e6089..dfc4efff9 100644
--- a/portality/crosswalks/oaipmh.py
+++ b/portality/crosswalks/oaipmh.py
@@ -490,10 +490,12 @@ def header(self, record):
 CROSSWALKS = {
     "oai_dc": {
         "article": OAI_DC_Article,
-        "journal": OAI_DC_Journal
+        "journal": OAI_DC_Journal,
+        "article,article_tombstone": OAI_DC_Article
     },
     'oai_doaj': {
-        "article": OAI_DOAJ_Article
+        "article": OAI_DOAJ_Article,
+        "article,article_tombstone": OAI_DOAJ_Article
     }
 }
 
diff --git a/portality/models/__init__.py b/portality/models/__init__.py
index 257092910..21f1f460a 100644
--- a/portality/models/__init__.py
+++ b/portality/models/__init__.py
@@ -14,7 +14,7 @@
 from portality.models.uploads import FileUpload, ExistsFileQuery, OwnerFileQuery, ValidFileQuery, BulkArticles
 from portality.models.lock import Lock
 from portality.models.history import ArticleHistory, JournalHistory
-from portality.models.article import Article, ArticleBibJSON, ArticleQuery, ArticleVolumesQuery, DuplicateArticleQuery, NoJournalException
+from portality.models.article import Article, ArticleBibJSON, ArticleQuery, ArticleVolumesQuery, DuplicateArticleQuery, NoJournalException, ArticleTombstone
 from portality.models.oaipmh import OAIPMHRecord, OAIPMHJournal, OAIPMHArticle
 from portality.models.atom import AtomRecord
 from portality.models.search import JournalArticle, JournalStatsQuery, ArticleStatsQuery
diff --git a/portality/models/article.py b/portality/models/article.py
index d431bb5ae..80f56fb2d 100644
--- a/portality/models/article.py
+++ b/portality/models/article.py
@@ -95,19 +95,26 @@ def delete_by_issns(cls, issns, snapshot=True):
         cls.delete_selected(query=q.query(), snapshot=snapshot)
 
     @classmethod
-    def delete_selected(cls, query=None, owner=None, snapshot=True):
+    def delete_selected(cls, query=None, owner=None, snapshot=True, tombstone=True):
         if owner is not None:
             from portality.models import Journal
             issns = Journal.issns_by_owner(owner)
             q = ArticleQuery(issns=issns)
             query = q.query()
 
-        if snapshot:
+        if snapshot or tombstone:
             articles = cls.iterate(query, page_size=1000)
             for article in articles:
-                article.snapshot()
+                if snapshot:
+                    article.snapshot()
+                if tombstone:
+                    article._tombstone()
         return cls.delete_by_query(query)
 
+    def delete(self):
+        self._tombstone()
+        super(Article, self).delete()
+
     def bibjson(self, **kwargs):
         if "bibjson" not in self.data:
             self.data["bibjson"] = {}
@@ -142,6 +149,18 @@ def snapshot(self):
         hist.save()
         return hist.id
 
+    def _tombstone(self):
+        stone = ArticleTombstone()
+        stone.set_id(self.id)
+        sbj = stone.bibjson()
+
+        subs = self.bibjson().subjects()
+        for s in subs:
+            sbj.add_subject(s.get("scheme"), s.get("term"), s.get("code"))
+
+        stone.save()
+        return stone
+
     def add_history(self, bibjson, date=None):
         """Deprecated"""
         bibjson = bibjson.bibjson if isinstance(bibjson, ArticleBibJSON) else bibjson
@@ -565,6 +584,23 @@ def get_owner(self):
 
         return owners[0]
 
+
+class ArticleTombstone(Article):
+    __type__ = "article_tombstone"
+
+    def snapshot(self):
+        return None
+
+    def is_in_doaj(self):
+        return False
+
+    def prep(self):
+        self.data['last_updated'] = dates.now_str()
+
+    def save(self, *args, **kwargs):
+        return super(ArticleTombstone, self).save(*args, **kwargs)
+
+
 class ArticleBibJSON(GenericBibJSON):
 
     def __init__(self, bibjson=None, **kwargs):
diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py
index 461cd9759..187426682 100644
--- a/portality/models/oaipmh.py
+++ b/portality/models/oaipmh.py
@@ -1,5 +1,6 @@
 from copy import deepcopy
-from portality.models import Journal, Article
+
+from portality.models import Journal, Article, ArticleTombstone
 from portality import constants
 
 class OAIPMHRecord(object):
@@ -114,17 +115,19 @@ def list_records(self, from_date=None, until_date=None, oai_set=None, list_size=
 
 
 class OAIPMHArticle(OAIPMHRecord, Article):
+    __type__ = "article,article_tombstone"
+
     def list_records(self, from_date=None, until_date=None, oai_set=None, list_size=None, start_after=None):
         total, results = super(OAIPMHArticle, self).list_records(from_date=from_date,
             until_date=until_date, oai_set=oai_set, list_size=list_size, start_after=start_after)
-        return total, [Article(**r) for r in results]
-
-    # def pull(self, identifier):
-    #     # override the default pull, as we care about whether the item is in_doaj
-    #     record = super(OAIPMHArticle, self).pull(identifier)
-    #     if record is not None and record.is_in_doaj():
-    #         return record
-    #     return None
+        return total, [Article(**r) if r.get("es_type") == "article" else ArticleTombstone(**r) for r in results]
+
+    def pull(self, identifier):
+        # override the default pull, as we must check the tombstone record too
+        article = Article.pull(identifier)
+        if article is None:
+            article = ArticleTombstone.pull(identifier)
+        return article
 
 
 class OAIPMHJournal(OAIPMHRecord, Journal):
diff --git a/portality/settings.py b/portality/settings.py
index c9abf564c..446fdcfa9 100644
--- a/portality/settings.py
+++ b/portality/settings.py
@@ -707,6 +707,7 @@
 MAPPINGS['provenance'] = MAPPINGS["account"]    #~~->Provenance:Model~~
 MAPPINGS['preserve'] = MAPPINGS["account"]    #~~->Preservation:Model~~
 MAPPINGS['notification'] = MAPPINGS["account"]    #~~->Notification:Model~~
+MAPPINGS['article_tombstone'] = MAPPINGS["account"]    #~~->ArticleTombstone:Model~~
 
 #########################################
 # Query Routes

From 01bf01175190e10d09f0f54886927f41e115ed04 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Mon, 16 Sep 2024 14:58:15 +0100
Subject: [PATCH 52/70] tidy up code for PR

---
 doajtest/helpers.py          |  1 -
 doajtest/unit/test_oaipmh.py |  5 +----
 portality/models/oaipmh.py   | 11 +----------
 3 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/doajtest/helpers.py b/doajtest/helpers.py
index 19c4629b2..fe2f585fa 100644
--- a/doajtest/helpers.py
+++ b/doajtest/helpers.py
@@ -21,7 +21,6 @@
 from portality.lib.thread_utils import wait_until
 from portality.tasks.redis_huey import main_queue, long_running
 from portality.util import url_for
-from view.admin import index
 
 
 def patch_config(inst, properties):
diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py
index 55f27467c..d5a3291de 100644
--- a/doajtest/unit/test_oaipmh.py
+++ b/doajtest/unit/test_oaipmh.py
@@ -10,7 +10,6 @@
 from doajtest.fixtures import ArticleFixtureFactory
 from doajtest.fixtures import JournalFixtureFactory
 from doajtest.helpers import DoajTestCase
-from models import ArticleTombstone
 from portality import models
 from portality.app import app
 from portality.lib import dates
@@ -54,7 +53,6 @@ def test_02_oai_journals(self):
         j_private = models.Journal(**journal_sources[1])
         j_private.set_in_doaj(False)
         j_private.save(blocking=True)
-        deleted_id = j_private.id
 
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
@@ -342,7 +340,6 @@ def test_09_article(self):
         stone.set_id(stone.makeid())
         stone.bibjson().add_subject("LCC", "Economic theory. Demography", "AB22")
         stone.save(blocking=True)
-        stone_id = stone.id
 
         models.Article.blockall([(a_private.id, a_private.last_updated), (a_public.id, a_public.last_updated)])
         models.ArticleTombstone.blockall([(stone.id, stone.last_updated)])
@@ -355,7 +352,7 @@ def test_09_article(self):
                 t = etree.fromstring(resp.data)
                 records = t.xpath('/oai:OAI-PMH/oai:ListRecords', namespaces=self.oai_ns)
 
-                # Check we only have two articles returned
+                # Check we only have three articles returned
                 r = records[0].xpath('//oai:record', namespaces=self.oai_ns)
                 assert len(r) == 3
 
diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py
index 187426682..113c6076e 100644
--- a/portality/models/oaipmh.py
+++ b/portality/models/oaipmh.py
@@ -42,9 +42,7 @@ class OAIPMHRecord(object):
         "track_total_hits": True,
         "query": {
             "bool": {
-                "must": [
-                    # { "term": { "admin.in_doaj": True } }
-                ]
+                "must": []
             }
         },
         "from": 0,
@@ -135,10 +133,3 @@ def list_records(self, from_date=None, until_date=None, oai_set=None, list_size=
         total, results = super(OAIPMHJournal, self).list_records(from_date=from_date,
             until_date=until_date, oai_set=oai_set, list_size=list_size, start_after=start_after)
         return total, [Journal(**r) for r in results]
-
-    # def pull(self, identifier):
-    #     # override the default pull, as we care about whether the item is in_doaj
-    #     record = super(OAIPMHJournal, self).pull(identifier)
-    #     if record is not None and record.is_in_doaj():
-    #         return record
-    #     return None

From b437581e5928dd0aebc1317a1a0b58240a5ac2c6 Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Fri, 4 Oct 2024 12:04:35 +0100
Subject: [PATCH 53/70] convert DOI to DO

---
 doajtest/unit/test_crosswalks_article_ris.py | 2 +-
 portality/crosswalks/article_ris.py          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doajtest/unit/test_crosswalks_article_ris.py b/doajtest/unit/test_crosswalks_article_ris.py
index 5d58655f4..90a4e3e6d 100644
--- a/doajtest/unit/test_crosswalks_article_ris.py
+++ b/doajtest/unit/test_crosswalks_article_ris.py
@@ -28,7 +28,7 @@ def test_article2ris(self):
 AB  - abstract
 KW  - word
 KW  - key
-DOI  - 10.0000/SOME.IDENTIFIER
+DO  - 10.0000/SOME.IDENTIFIER
 ER  - 
         """.split()
 
diff --git a/portality/crosswalks/article_ris.py b/portality/crosswalks/article_ris.py
index 214720084..c15cfcffa 100644
--- a/portality/crosswalks/article_ris.py
+++ b/portality/crosswalks/article_ris.py
@@ -24,7 +24,7 @@ def extra_author_names(article) -> list:
     'UR': '$.bibjson.link[*].url',
     'AB': '$.bibjson.abstract',
     'KW': '$.bibjson.keywords[*]',
-    'DOI': '$.bibjson.identifier[?(@.type == "doi")].id',
+    'DO': '$.bibjson.identifier[?(@.type == "doi")].id',
     'SN': '$.bibjson.journal.issns[*]',
     'LA': '$.language.language',
 }

From bdac4cc640774d452e3f245151d26988723c02ee Mon Sep 17 00:00:00 2001
From: philip <philip.k@cottagelabs.com>
Date: Fri, 4 Oct 2024 12:50:48 +0100
Subject: [PATCH 54/70] update test cases for languages

---
 doajtest/unit/test_crosswalks_article_ris.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doajtest/unit/test_crosswalks_article_ris.py b/doajtest/unit/test_crosswalks_article_ris.py
index 90a4e3e6d..760d819e1 100644
--- a/doajtest/unit/test_crosswalks_article_ris.py
+++ b/doajtest/unit/test_crosswalks_article_ris.py
@@ -29,6 +29,8 @@ def test_article2ris(self):
 KW  - word
 KW  - key
 DO  - 10.0000/SOME.IDENTIFIER
+LA  - EN
+LA  - FR
 ER  - 
         """.split()
 

From d57f619c74f04ef8adde354ed4db6929a69d64d1 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Thu, 24 Oct 2024 13:32:08 +0100
Subject: [PATCH 55/70] exclude on-hold items from all rules, not just maned
 rules, as these sometimes cause maneds to see applications under 'new
 application' which are also 'on hold'

---
 portality/bll/services/todo.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py
index cd3723bb3..25a62a2c7 100644
--- a/portality/bll/services/todo.py
+++ b/portality/bll/services/todo.py
@@ -336,7 +336,8 @@ def editor_stalled(cls, groups, size):
                 TodoQuery.status([
                     constants.APPLICATION_STATUS_ACCEPTED,
                     constants.APPLICATION_STATUS_REJECTED,
-                    constants.APPLICATION_STATUS_READY
+                    constants.APPLICATION_STATUS_READY,
+                    constants.APPLICATION_STATUS_ON_HOLD
                 ])
             ],
             sort=sort_date,
@@ -357,7 +358,8 @@ def editor_follow_up_old(cls, groups, size):
                 TodoQuery.status([
                     constants.APPLICATION_STATUS_ACCEPTED,
                     constants.APPLICATION_STATUS_REJECTED,
-                    constants.APPLICATION_STATUS_READY
+                    constants.APPLICATION_STATUS_READY,
+                    constants.APPLICATION_STATUS_ON_HOLD
                 ])
             ],
             sort=sort_date,
@@ -410,7 +412,8 @@ def associate_stalled(cls,  acc_id, size):
                     constants.APPLICATION_STATUS_ACCEPTED,
                     constants.APPLICATION_STATUS_REJECTED,
                     constants.APPLICATION_STATUS_READY,
-                    constants.APPLICATION_STATUS_COMPLETED
+                    constants.APPLICATION_STATUS_COMPLETED,
+                    constants.APPLICATION_STATUS_ON_HOLD
                 ])
             ],
             sort=sort_field,
@@ -432,7 +435,8 @@ def associate_follow_up_old(cls,  acc_id, size):
                     constants.APPLICATION_STATUS_ACCEPTED,
                     constants.APPLICATION_STATUS_REJECTED,
                     constants.APPLICATION_STATUS_READY,
-                    constants.APPLICATION_STATUS_COMPLETED
+                    constants.APPLICATION_STATUS_COMPLETED,
+                    constants.APPLICATION_STATUS_ON_HOLD
                 ])
             ],
             sort=sort_field,
@@ -467,7 +471,8 @@ def associate_all_applications(cls, acc_id, size):
                     constants.APPLICATION_STATUS_ACCEPTED,
                     constants.APPLICATION_STATUS_REJECTED,
                     constants.APPLICATION_STATUS_READY,
-                    constants.APPLICATION_STATUS_COMPLETED
+                    constants.APPLICATION_STATUS_COMPLETED,
+                    constants.APPLICATION_STATUS_ON_HOLD
                 ])
             ],
             sort=sort_field,

From b8dd4d24021c2f7572083a18d5873e7e9a5633c2 Mon Sep 17 00:00:00 2001
From: Steven Eardley <steve@eardley.xyz>
Date: Tue, 5 Nov 2024 12:22:35 +0000
Subject: [PATCH 56/70] reinstate CI on release branches

---
 .circleci/config.yml  | 1 -
 portality/settings.py | 2 +-
 setup.py              | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index f08608285..a0a154163 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -74,5 +74,4 @@ workflows:
           filters:
             branches:
               ignore:
-                - /release\/.*/
                 - static_pages
diff --git a/portality/settings.py b/portality/settings.py
index 201d65edf..213961706 100644
--- a/portality/settings.py
+++ b/portality/settings.py
@@ -9,7 +9,7 @@
 # Application Version information
 # ~~->API:Feature~~
 
-DOAJ_VERSION = "7.0.0"
+DOAJ_VERSION = "7.0.1"
 API_VERSION = "4.0.0"
 
 ######################################
diff --git a/setup.py b/setup.py
index 7682c79b2..8fbc87f66 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 
 setup(
     name='doaj',
-    version='7.0.0',
+    version='7.0.1',
     packages=find_packages(),
     install_requires=[
         "awscli==1.20.50",

From ebc40714f1c7b4cb20e9c4e37cb8b743d48b419c Mon Sep 17 00:00:00 2001
From: Steven Eardley <steve@eardley.xyz>
Date: Tue, 5 Nov 2024 14:53:20 +0000
Subject: [PATCH 57/70] Fix leaky test for OAI-PMH endpoint

---
 doajtest/unit/test_oaipmh.py | 92 +++++++++++++++++++++++-------------
 portality/models/oaipmh.py   | 15 +++---
 2 files changed, 68 insertions(+), 39 deletions(-)

diff --git a/doajtest/unit/test_oaipmh.py b/doajtest/unit/test_oaipmh.py
index d5a3291de..f74569ad6 100644
--- a/doajtest/unit/test_oaipmh.py
+++ b/doajtest/unit/test_oaipmh.py
@@ -18,10 +18,15 @@
 
 from doajtest.helpers import with_es
 
+
 class TestClient(DoajTestCase):
     @classmethod
     def setUpClass(cls):
         app.testing = True
+
+        # Preserve default value of OAI record page size
+        cls.DEFAULT_OAIPMH_LIST_IDENTIFIERS_PAGE_SIZE = app.config.get("OAIPMH_LIST_IDENTIFIERS_PAGE_SIZE", 25)
+
         super(TestClient, cls).setUpClass()
 
     def setUp(self):
@@ -31,7 +36,11 @@ def setUp(self):
         self.oai_ns = {'oai': 'http://www.openarchives.org/OAI/2.0/',
                        'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
                        'dc': 'http://purl.org/dc/elements/1.1/',
-                       'xsi' : 'http://www.w3.org/2001/XMLSchema-instance'}
+                       'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
+
+    def tearDown(self):
+        app.config['OAIPMH_LIST_IDENTIFIERS_PAGE_SIZE'] = self.DEFAULT_OAIPMH_LIST_IDENTIFIERS_PAGE_SIZE
+        super(TestClient, self).tearDown()
 
     def test_01_oai_ListMetadataFormats(self):
         """ Check we get the correct response from the OAI endpoint ListMetdataFormats request"""
@@ -41,7 +50,8 @@ def test_01_oai_ListMetadataFormats(self):
                 assert resp.status_code == 200
 
                 t = etree.fromstring(resp.data)
-                assert t.xpath('/oai:OAI-PMH/oai:ListMetadataFormats/oai:metadataFormat/oai:metadataPrefix', namespaces=self.oai_ns)[0].text == 'oai_dc'
+                assert t.xpath('/oai:OAI-PMH/oai:ListMetadataFormats/oai:metadataFormat/oai:metadataPrefix',
+                               namespaces=self.oai_ns)[0].text == 'oai_dc'
 
     def test_02_oai_journals(self):
         """test if the OAI-PMH journal feed returns records and only displays journals accepted in DOAJ, marking withdrawn ones as deleted"""
@@ -81,7 +91,9 @@ def test_02_oai_journals(self):
                 assert seen_deleted
                 assert seen_public
 
-                resp = t_client.get(url_for('oaipmh.oaipmh', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier={0}'.format(public_id))
+                resp = t_client.get(
+                    url_for('oaipmh.oaipmh', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier={0}'.format(
+                        public_id))
                 assert resp.status_code == 200
 
                 t = etree.fromstring(resp.data)
@@ -123,7 +135,7 @@ def test_03_oai_resumption_token(self):
             with self.app_test.test_client() as t_client:
                 resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListIdentifiers', metadataPrefix='oai_dc'))
                 t = etree.fromstring(resp.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
                 rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0]
                 assert rt.get('completeListSize') == '5'
                 assert rt.get('cursor') == '2'
@@ -131,7 +143,7 @@ def test_03_oai_resumption_token(self):
                 # Get the next result
                 resp2 = t_client.get(url_for('oaipmh.oaipmh', verb='ListIdentifiers', resumptionToken=rt.text))
                 t = etree.fromstring(resp2.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
                 rt2 = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0]
                 assert rt2.get('completeListSize') == '5'
                 assert rt2.get('cursor') == '4'
@@ -139,17 +151,18 @@ def test_03_oai_resumption_token(self):
                 # And the final result - check we get an empty resumptionToken
                 resp3 = t_client.get(url_for('oaipmh.oaipmh', verb='ListIdentifiers', resumptionToken=rt2.text))
                 t = etree.fromstring(resp3.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
                 rt3 = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0]
                 assert rt3.get('completeListSize') == '5'
                 assert rt3.get('cursor') == '5'
                 assert rt3.text is None
 
                 # We should get an error if we request again with an empty resumptionToken
-                resp4 = t_client.get(url_for('oaipmh.oaipmh', verb='ListIdentifiers') + '&resumptionToken={0}'.format(rt3.text))
-                assert resp4.status_code == 200                                   # fixme: should this be a real error code?
+                resp4 = t_client.get(
+                    url_for('oaipmh.oaipmh', verb='ListIdentifiers') + '&resumptionToken={0}'.format(rt3.text))
+                assert resp4.status_code == 200  # fixme: should this be a real error code?
                 t = etree.fromstring(resp4.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
 
                 err = t.xpath('//oai:error', namespaces=self.oai_ns)[0]
                 assert 'the resumptionToken argument is invalid or expired' in err.text
@@ -171,9 +184,11 @@ def test_04_oai_changing_index(self):
         yesterday = (dates.now() - timedelta(days=1)).strftime(FMT_DATE_STD)
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
-                resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(yesterday))
+                resp = t_client.get(
+                    url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(
+                        yesterday))
                 t = etree.fromstring(resp.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
                 rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0]
                 assert rt.get('completeListSize') == '3'
                 assert rt.get('cursor') == '2'
@@ -187,15 +202,17 @@ def test_04_oai_changing_index(self):
                 resp2 = t_client.get('/oai?verb=ListRecords&resumptionToken={0}'.format(rt.text))
                 resp2 = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', resumptionToken=rt.text))
                 t = etree.fromstring(resp2.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
                 rt2 = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0]
                 assert rt2.get('completeListSize') == '3'
                 assert rt2.get('cursor') == '3'
 
                 # Start a new request - we should see the new journal
-                resp3 = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(yesterday))
+                resp3 = t_client.get(
+                    url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(
+                        yesterday))
                 t = etree.fromstring(resp3.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
                 rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0]
                 assert rt.get('completeListSize') == '4'
 
@@ -227,9 +244,11 @@ def test_05_date_ranges(self):
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
                 # Request OAI journals since yesterday (looking for today's results only)
-                resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(yesterday.strftime(FMT_DATE_STD)))
+                resp = t_client.get(
+                    url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}'.format(
+                        yesterday.strftime(FMT_DATE_STD)))
                 t = etree.fromstring(resp.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
                 rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0]
                 assert rt.get('completeListSize') == '2'
                 assert rt.get('cursor') == '1'
@@ -238,10 +257,11 @@ def test_05_date_ranges(self):
                     assert title.text in [journals[2]['bibjson']['title'], journals[3]['bibjson']['title']]
 
                 # Request OAI journals from 3 days ago to yesterday (expecting the 2 days ago results)
-                resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc') + '&from={0}&until={1}'.format(
+                resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords',
+                                            metadataPrefix='oai_dc') + '&from={0}&until={1}'.format(
                     two_days_before_yesterday.strftime(FMT_DATE_STD), yesterday.strftime(FMT_DATE_STD)))
                 t = etree.fromstring(resp.data)
-                #print etree.tostring(t, pretty_print=True)
+                # print etree.tostring(t, pretty_print=True)
                 rt = t.xpath('//oai:resumptionToken', namespaces=self.oai_ns)[0]
                 assert rt.get('completeListSize') == '2'
                 assert rt.get('cursor') == '1'
@@ -262,7 +282,8 @@ def test_06_identify(self):
                 t = etree.fromstring(resp.data)
                 records = t.xpath('/oai:OAI-PMH/oai:Identify', namespaces=self.oai_ns)
             assert len(records) == 1
-            assert records[0].xpath('//oai:repositoryName', namespaces=self.oai_ns)[0].text == 'Directory of Open Access Journals'
+            assert records[0].xpath('//oai:repositoryName', namespaces=self.oai_ns)[
+                       0].text == 'Directory of Open Access Journals'
             assert records[0].xpath('//oai:adminEmail', namespaces=self.oai_ns)[0].text == 'helpdesk+oai@doaj.org'
             assert records[0].xpath('//oai:granularity', namespaces=self.oai_ns)[0].text == 'YYYY-MM-DDThh:mm:ssZ'
 
@@ -278,15 +299,17 @@ def test_07_bad_verb(self):
                 assert resp.status_code == 200
                 t = etree.fromstring(resp.data)
                 records = t.xpath('/oai:OAI-PMH', namespaces=self.oai_ns)
-                assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[0].text == 'Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated.'
+                assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[
+                           0].text == 'Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated.'
                 assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[0].get("code") == 'badVerb'
 
-                #invalid verb
+                # invalid verb
                 resp = t_client.get(url_for('oaipmh.oaipmh', verb='InvalidVerb', metadataPrefix='oai_dc'))
                 assert resp.status_code == 200
                 t = etree.fromstring(resp.data)
                 records = t.xpath('/oai:OAI-PMH', namespaces=self.oai_ns)
-                assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[0].text == 'Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated.'
+                assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[
+                           0].text == 'Value of the verb argument is not a legal OAI-PMH verb, the verb argument is missing, or the verb argument is repeated.'
                 assert records[0].xpath('//oai:error', namespaces=self.oai_ns)[0].get("code") == 'badVerb'
 
     def test_08_list_sets(self):
@@ -311,14 +334,14 @@ def test_08_list_sets(self):
 
             # check that we can retrieve a record with one of those sets
             with self.app_test.test_client() as t_client:
-                resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set=set0[0].text))
+                resp = t_client.get(
+                    url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set=set0[0].text))
                 assert resp.status_code == 200
                 t = etree.fromstring(resp.data)
                 records = t.xpath('/oai:OAI-PMH/oai:ListRecords', namespaces=self.oai_ns)
                 results = records[0].getchildren()
             assert len(results) == 1
 
-
     def test_09_article(self):
         """test if the OAI-PMH article feed returns records and only displays articles accepted in DOAJ, showing the others as deleted"""
         article_source = ArticleFixtureFactory.make_article_source(eissn='1234-1234', pissn='5678-5678,', in_doaj=False)
@@ -346,7 +369,8 @@ def test_09_article(self):
 
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
-                resp = t_client.get(url_for('oaipmh.oaipmh',  specified='article', verb='ListRecords', metadataPrefix='oai_dc'))
+                resp = t_client.get(
+                    url_for('oaipmh.oaipmh', specified='article', verb='ListRecords', metadataPrefix='oai_dc'))
                 assert resp.status_code == 200
 
                 t = etree.fromstring(resp.data)
@@ -378,7 +402,8 @@ def test_09_article(self):
                 assert seen_deleted == 2
                 assert seen_public
 
-                resp = t_client.get(url_for('oaipmh.oaipmh',  specified='article', verb='GetRecord', metadataPrefix='oai_dc') + '&identifier=' + public_id)
+                resp = t_client.get(url_for('oaipmh.oaipmh', specified='article', verb='GetRecord',
+                                            metadataPrefix='oai_dc') + '&identifier=' + public_id)
                 assert resp.status_code == 200
 
                 t = etree.fromstring(resp.data)
@@ -402,7 +427,8 @@ def test_10_subjects(self):
         with self.app_test.test_request_context():
             # Check whether the journal is found for its specific set: Veterinary Medicine (TENDOlZldGVyaW5hcnkgbWVkaWNpbmU)
             with self.app_test.test_client() as t_client:
-                resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set='TENDOlZldGVyaW5hcnkgbWVkaWNpbmU~'))
+                resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc',
+                                            set='TENDOlZldGVyaW5hcnkgbWVkaWNpbmU~'))
                 assert resp.status_code == 200
 
                 t = etree.fromstring(resp.data)
@@ -414,7 +440,7 @@ def test_10_subjects(self):
                 # Check we have the correct journal
                 assert records[0].xpath('//dc:title', namespaces=self.oai_ns)[0].text == j_public.bibjson().title
 
-                #check we have expected subjects (Veterinary Medicine but not Agriculture)
+                # check we have expected subjects (Veterinary Medicine but not Agriculture)
                 subjects = records[0].xpath('//dc:subject', namespaces=self.oai_ns)
                 assert len(subjects) != 0
 
@@ -426,7 +452,8 @@ def test_10_subjects(self):
         with self.app_test.test_request_context():
             # Check whether the journal is found for more general set: Agriculture (TENDOkFncmljdWx0dXJl)
             with self.app_test.test_client() as t_client:
-                resp = t_client.get(url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set='TENDOkFncmljdWx0dXJl~'))
+                resp = t_client.get(
+                    url_for('oaipmh.oaipmh', verb='ListRecords', metadataPrefix='oai_dc', set='TENDOkFncmljdWx0dXJl~'))
                 assert resp.status_code == 200
 
                 t = etree.fromstring(resp.data)
@@ -457,7 +484,8 @@ def test_11_oai_dc_attr(self):
 
         with self.app_test.test_request_context():
             with self.app_test.test_client() as t_client:
-                resp = t_client.get(url_for('oaipmh.oaipmh',  specified='article', verb='ListRecords', metadataPrefix='oai_dc'))
+                resp = t_client.get(
+                    url_for('oaipmh.oaipmh', specified='article', verb='ListRecords', metadataPrefix='oai_dc'))
                 assert resp.status_code == 200
 
                 t = etree.fromstring(resp.data)
@@ -483,7 +511,7 @@ def test_11_oai_dc_attr(self):
                 t = etree.fromstring(resp.data)
                 # find metadata element of our record
                 elem = t.xpath('/oai:OAI-PMH/oai:ListRecords/oai:record/oai:metadata', namespaces=self.oai_ns)
-                #metadata element should have only one child, "dc" with correct nsmap
+                # metadata element should have only one child, "dc" with correct nsmap
                 oai_dc = elem[0].getchildren()
                 assert len(oai_dc) == 1
                 assert oai_dc[0].tag == "{%s}" % self.oai_ns["oai_dc"] + "dc"
@@ -499,4 +527,4 @@ def test_decode_resumption_token__fail(self):
 
     def test_decode_resumption_token(self):
         params = decode_resumption_token(base64.urlsafe_b64encode(b'{"m":1}').decode('utf-8'))
-        assert params == {"metadata_prefix": 1}
\ No newline at end of file
+        assert params == {"metadata_prefix": 1}
diff --git a/portality/models/oaipmh.py b/portality/models/oaipmh.py
index 113c6076e..e7c050fe6 100644
--- a/portality/models/oaipmh.py
+++ b/portality/models/oaipmh.py
@@ -3,17 +3,18 @@
 from portality.models import Journal, Article, ArticleTombstone
 from portality import constants
 
+
 class OAIPMHRecord(object):
     earliest = {
         "query": {
             "bool": {
                 "must": [
-                    { "term": { "admin.in_doaj": True } }
+                    {"term": {"admin.in_doaj": True}}
                 ]
             }
         },
         "size": 1,
-        "sort" : [
+        "sort": [
             {"last_updated": {"order": "asc"}}
         ]
     }
@@ -22,7 +23,7 @@ class OAIPMHRecord(object):
         "query": {
             "bool": {
                 "must": [
-                    { "term": { "admin.in_doaj": True } }
+                    {"term": {"admin.in_doaj": True}}
                 ]
             }
         },
@@ -31,7 +32,7 @@ class OAIPMHRecord(object):
             "sets": {
                 "terms": {
                     "field": "index.schema_subject.exact",
-                    "order": {"_key" : "asc"},
+                    "order": {"_key": "asc"},
                     "size": 100000
                 }
             }
@@ -49,9 +50,9 @@ class OAIPMHRecord(object):
         "size": 25
     }
 
-    set_limit = {"term" : { "index.classification.exact" : "<set name>" }}
-    range_limit = { "range" : { "last_updated" : {"gte" : "<from date>", "lte" : "<until date>"} } }
-    created_sort = [{"last_updated" : {"order" : "desc"}}, {"id.exact" : "desc"}]
+    set_limit = {"term": {"index.classification.exact": "<set name>"}}
+    range_limit = {"range": {"last_updated": {"gte": "<from date>", "lte": "<until date>"}}}
+    created_sort = [{"last_updated": {"order": "desc"}}, {"id.exact": "desc"}]
 
     def earliest_datestamp(self):
         result = self.query(q=self.earliest)

From 270a655a7a2a30cb6292bad6262860ce7fef20f0 Mon Sep 17 00:00:00 2001
From: Steven Eardley <steve@eardley.xyz>
Date: Tue, 5 Nov 2024 20:49:17 +0000
Subject: [PATCH 58/70] add CI badge to readme

---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index bb04445ec..7b21709af 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,11 @@
 This repository provides the software which drives the DOAJ website and the DOAJ 
 directory.
 
+## CI Status
+
+**develop** [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/develop.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/develop)
+**master** [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/master)
+
 ## Reporting issues
 
 Please feel free to use the issue tracker on https://github.com/DOAJ/doaj/issues for any bug 

From 74ee3a4c0b5f2488bfd0644bf14fbee0c89c521a Mon Sep 17 00:00:00 2001
From: Steven Eardley <steve@eardley.xyz>
Date: Tue, 5 Nov 2024 20:51:54 +0000
Subject: [PATCH 59/70] A bit for formatting for README

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7b21709af..e8c8c4877 100644
--- a/README.md
+++ b/README.md
@@ -6,8 +6,9 @@ directory.
 
 ## CI Status
 
-**develop** [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/develop.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/develop)
-**master** [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/master)
+**develop** &nbsp; [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/develop.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/develop)
+
+**master** &nbsp; [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DOAJ/doaj/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DOAJ/doaj/tree/master)
 
 ## Reporting issues
 

From e5f89f28ffc99336f91b2af55b2ec48f3fd464cf Mon Sep 17 00:00:00 2001
From: Dom Mitchell <dommitchell@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:48:58 +0100
Subject: [PATCH 60/70] Edited l.11

---
 .../_application-form/includes/_fieldset_oa_compliance.html     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/portality/templates-v2/_application-form/includes/_fieldset_oa_compliance.html b/portality/templates-v2/_application-form/includes/_fieldset_oa_compliance.html
index 932e2b395..b0170c9ba 100644
--- a/portality/templates-v2/_application-form/includes/_fieldset_oa_compliance.html
+++ b/portality/templates-v2/_application-form/includes/_fieldset_oa_compliance.html
@@ -8,7 +8,7 @@
     <ol>
       <li>The application form takes approximately <strong>30 minutes</strong> to complete.</li>
       <li>Your progress is automatically saved.</li>
-      <li>You can return to this application at any time by clicking <a href="{{ url_for("publisher.index") }}" target="_blank"><em>My account</em> &rarr; <em>Publisher</em></a> at the top.</li>
+      <li>You can return to this application at any time by clicking <a href="{{ url_for("publisher.index") }}" target="_blank"><em>Dashboard</em> &rarr; <em>Publisher dashboard</em></a> at the top.</li>
       <li>You can <a href="/static/doaj/docs/DOAJquestions-for-reference-only.pdf" target="_blank">print or download</a> a PDF list of the questions.</li>
       <li>You must apply online.</li>
     </ol>

From 6077811c1e943d99a1ca219a029e89d251e0411c Mon Sep 17 00:00:00 2001
From: Dom Mitchell <dommitchell@users.noreply.github.com>
Date: Thu, 7 Nov 2024 15:18:34 +0100
Subject: [PATCH 61/70] Updated revision history for new delete flag

---
 cms/pages/docs/oai-pmh.md | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/cms/pages/docs/oai-pmh.md b/cms/pages/docs/oai-pmh.md
index 8eac6f9b3..8d5124c73 100644
--- a/cms/pages/docs/oai-pmh.md
+++ b/cms/pages/docs/oai-pmh.md
@@ -11,17 +11,19 @@ featuremap:
 
 ---
 
+Journal records are announced in the feed when they are added or removed.
+
 ## Journal feed
 
 {:.tabular-list}
 - `Identify`
   - Access the [base Identify endpoint](/oai?verb=Identify).
 - `ListSets`
-  - DOAJ provides all its subject classifications as OAI-PMH sets, so you can harvest just those you are interested in. Access the [full list of the sets](/oai?verb=ListSets).
+  - We provide all our subject classifications as OAI-PMH sets, so you can harvest only those you are interested in. Access the [full list of the sets](/oai?verb=ListSets).
 - `ListMetadataFormats`
-  - DOAJ currently supports only `oai_dc`; access [the metadata formats](/oai?verb=ListMetadataFormats).
+  - Currently, we only support `oai_dc`; access [the metadata formats](/oai?verb=ListMetadataFormats).
 
-The metadata held by DOAJ is mapped to Dublin Core in the OAI-PMH feed, with the following interpretations for each Journal field:
+The metadata held by us is mapped to Dublin Core in the OAI-PMH feed, with the following interpretations for each Journal field:
 
 | Dublin Core   | Meaning within DOAJ                                                                                                                                                                                                                                                                                      |
 |---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -91,6 +93,7 @@ The metadata held by DOAJ is mapped to Dublin Core in the OAI-PMH feed, with the
 
 | Date changes were made live | Changes                                                                                                                                                                                                                                                 |
 |-----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 6 November 2024             | added support for article and journal delete to the base Identify endpoint |
 | 25 November 2020            | a minor edit to `dc:relation`: 'Links to related resources: the journal home page and the journal author-pays link if relevant' became 'Links to related resources (if present): the journal home page, open access statement, author instructions, aims, and waiver pages'|
 | 20 April 2015               | `subject` elements which represent a Library of Congress Classification \(LCC\) topic will now be marked with an additional OAI DC\-compliant attribute to denote this: `xsi:type="dcterms:LCSH"`\. LCC subjects will no longer be prefixed by `LCC:`\. |
 | 13 December 2013            | Initial release                                                                                                                                                                                                                                         |
@@ -103,13 +106,13 @@ The metadata held by DOAJ is mapped to Dublin Core in the OAI-PMH feed, with the
 - `Identify`
   - Access the [base Identify endpoint](http://www.doaj.org/oai.article?verb=Identify).
 - `ListSets`
-  - DOAJ provides all its subject classifications as OAI-PMH sets, so you can harvest just those you are interested in. Access the [full list of the sets](http://www.doaj.org/oai.article?verb=ListSets).
+  - We provide all our subject classifications as OAI-PMH sets, so you can harvest only those you are interested in. Access the [full list of the sets](http://www.doaj.org/oai.article?verb=ListSets).
 - `ListMetadataFormats`
-  - DOAJ currently supports the `oai_dc` and `oai_doaj` formats; access [the metadata formats](http://www.doaj.org/oai.article?verb=ListMetadataFormats).
+  - Currently, we only support the `oai_dc` and `oai_doaj` formats; access [the metadata formats](http://www.doaj.org/oai.article?verb=ListMetadataFormats).
 
 ### Dublin Core OAI Article format (`OAI_DC`)
 
-The metadata held by DOAJ is mapped to Dublin Core in the OAI-PMH feed, with the following interpretations for each Article field:
+The metadata held by us is mapped to Dublin Core in the OAI-PMH feed, with the following interpretations for each Article field:
 
 | Dublin Core   | Meaning within DOAJ                                                                                                                                                                                                                                                                                           |
 |---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -190,6 +193,7 @@ The metadata held by DOAJ is mapped to Dublin Core in the OAI-PMH feed, with the
 
 | Date changes were made live                                                                                                                                                                                                                       | Changes         |
 |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|
+| 6 November 2024             | added support for article and journal delete to the base Identify endpoint |
 | 25 November 2020                                                                                                                                                                                                                                  | The `dc:provenance` element was removed, as we no longer synchronise provenance information to our article records from the journal\. |
 | 22 August 2016                                                                                                                                                                                                                                    | The `dc:rights` element was removed, as it was technically inaccurate \- it represented the Journal's overall licence policy, not the specific rights for the article\. This information is now in `dc:provenance`\. |
 | 20 April 2015                                                                                                                                                                                                                                     | The `identifier` element will now point to the DOAJ article page rather than the `/search` page\. E\.g\. [`https://doaj.org/article/0000178c89214dc8b82df1a25c0c478e`](https://doaj.org/article/0000178c89214dc8b82df1a25c0c478e) <br/><br/>Up to two new `relation` elements will appear for each article, containing URL\-s to the Table of Contents page for the article's journal\. The page can be reached via both print ISSN and E\-ISSN, so up to two such links might appear\. <br/><br/>`subject` elements which represent a Library of Congress Classification \(LCC\) topic will now be marked with an additional OAI DC\-compliant attribute to denote this: `xsi:type="dcterms:LCSH"`\. LCC subjects will no longer be prefixed by `LCC:`\. |
@@ -288,4 +292,5 @@ The following fields are available (not every article will have all the informat
 
 | Date changes were made live | Changes         |
 |-----------------------------|-----------------|
+| 6 November 2024             | added support for article and journal delete to the base Identify endpoint |
 | 20 April 2015               | Initial release |

From a3957c73f6e2580d1b8fa20a0fd399a505b957de Mon Sep 17 00:00:00 2001
From: leenashah73 <65942894+leenashah73@users.noreply.github.com>
Date: Thu, 7 Nov 2024 14:33:19 +0000
Subject: [PATCH 62/70] Update notifications.yml

Edited the text for application:publisher:accepted:notify:  to add the 2 links to How to submit an UR and link to publisher page
---
 cms/data/notifications.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cms/data/notifications.yml b/cms/data/notifications.yml
index f514d7e4e..1940db4da 100644
--- a/cms/data/notifications.yml
+++ b/cms/data/notifications.yml
@@ -44,6 +44,10 @@ application:publisher:accepted:notify:
 
     It is your responsibility to keep the information about your journal in DOAJ up to date. When there are changes or updates needed please [submit an Update Request](https://doaj.org/publisher/journal) from your Publisher dashboard promptly. Please be aware that failure to do this may result in removal of your journal from DOAJ.
 
+    [How to submit an Update Request](https://doaj.org/apply/publisher-responsibilities/#keeping-your-journal-records-up-to-date)    
+
+    For more information on managing your DOAJ account and journal records, see our Publisher Information page (https://doaj.org/apply/publisher-responsibilities)                                                                                                                                                                                
+
     To increase the visibility, distribution and usage of your journal content, we encourage you to upload article metadata for this journal to DOAJ as soon as possible.
 
     [How to upload article metadata]({faq_url}#uploading-article-metadata)

From 4054144ee2e664b3f4ec45ce337326add0919a4d Mon Sep 17 00:00:00 2001
From: Dom Mitchell <dommitchell@users.noreply.github.com>
Date: Tue, 12 Nov 2024 10:14:56 +0100
Subject: [PATCH 63/70] Update oai-pmh.md

---
 cms/pages/docs/oai-pmh.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cms/pages/docs/oai-pmh.md b/cms/pages/docs/oai-pmh.md
index 8d5124c73..443bc8d9d 100644
--- a/cms/pages/docs/oai-pmh.md
+++ b/cms/pages/docs/oai-pmh.md
@@ -93,7 +93,7 @@ The metadata held by us is mapped to Dublin Core in the OAI-PMH feed, with the f
 
 | Date changes were made live | Changes                                                                                                                                                                                                                                                 |
 |-----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 6 November 2024             | added support for article and journal delete to the base Identify endpoint |
+| 6 November 2024             | added support for article and journal delete to the base Identify endpoint. (Sponsored by Clarivate.) |
 | 25 November 2020            | a minor edit to `dc:relation`: 'Links to related resources: the journal home page and the journal author-pays link if relevant' became 'Links to related resources (if present): the journal home page, open access statement, author instructions, aims, and waiver pages'|
 | 20 April 2015               | `subject` elements which represent a Library of Congress Classification \(LCC\) topic will now be marked with an additional OAI DC\-compliant attribute to denote this: `xsi:type="dcterms:LCSH"`\. LCC subjects will no longer be prefixed by `LCC:`\. |
 | 13 December 2013            | Initial release                                                                                                                                                                                                                                         |

From 3747ce94df5c98167247b66645544650d50ca73d Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Tue, 12 Nov 2024 11:50:58 +0000
Subject: [PATCH 64/70] disable notifications requested in
 https://github.com/DOAJ/doajPM/issues/3974

---
 doajtest/functional/make_notifications.py | 81 ++++++++++++-----------
 portality/bll/services/events.py          | 26 ++++----
 portality/forms/application_processors.py | 10 ---
 portality/ui/messages.py                  |  4 --
 4 files changed, 55 insertions(+), 66 deletions(-)

diff --git a/doajtest/functional/make_notifications.py b/doajtest/functional/make_notifications.py
index 3d5aeea0c..64aa044b5 100644
--- a/doajtest/functional/make_notifications.py
+++ b/doajtest/functional/make_notifications.py
@@ -6,6 +6,7 @@
 from portality import constants
 from portality import models, app_email
 from portality.core import app
+from portality.bll import DOAJ
 from portality.events.consumers import application_assed_assigned_notify, \
     application_assed_inprogress_notify, \
     application_editor_completed_notify, \
@@ -30,34 +31,36 @@
 
 USER = "richard"
 
-NOTIFICATIONS = [
-    "application_assed_assigned_notify",
-    "application_assed_inprogress_notify",
-    "application_editor_completed_notify",
-    "application_editor_group_assigned_notify",
-    "application_editor_inprogress_notify",
-    "application_maned_ready_notify",
-    "application_publisher_accepted_notify",
-    "application_publisher_assigned_notify",
-    "application_publisher_created_notify",
-    "application_publisher_inprogress_notify",
-    "application_publisher_quickreject_notify",
-    "application_publisher_revision_notify",
-    "bg_job_finished_notify",
-    "journal_assed_assigned_notify",
-    "journal_editor_group_assigned_notify",
-    "update_request_publisher_accepted_notify",
-    "update_request_publisher_assigned_notify",
-    "update_request_publisher_rejected_notify",
-    UpdateRequestPublisherSubmittedNotify.ID,
-]
+NOTIFICATIONS = [ec.ID for ec in DOAJ.eventsService().EVENT_CONSUMERS]
+
+# NOTIFICATIONS = [
+#     "application_assed_assigned_notify",
+#     "application_assed_inprogress_notify",
+#     "application_editor_completed_notify",
+#     "application_editor_group_assigned_notify",
+#     "application_editor_inprogress_notify",
+#     "application_maned_ready_notify",
+#     "application_publisher_accepted_notify",
+#     "application_publisher_assigned_notify",
+#     "application_publisher_created_notify",
+#     "application_publisher_inprogress_notify",
+#     "application_publisher_quickreject_notify",
+#     "application_publisher_revision_notify",
+#     "bg_job_finished_notify",
+#     "journal_assed_assigned_notify",
+#     "journal_editor_group_assigned_notify",
+#     "update_request_publisher_accepted_notify",
+#     "update_request_publisher_assigned_notify",
+#     "update_request_publisher_rejected_notify",
+#     UpdateRequestPublisherSubmittedNotify.ID,
+# ]
 
 app.config["ENABLE_EMAIL"] = True
 app_email.Mail = MockMail
 
 ##############################################
 ## ApplicationAssedAssignedNotify
-if "application_assed_assigned_notify" in NOTIFICATIONS:
+if "application:assed:assigned:notify" in NOTIFICATIONS:
     aaan_application = ApplicationFixtureFactory.make_application_source()
     aaan_application["admin"]["editor"] = USER
     aaan_application["bibjson"]["title"] = "Application Assed Assigned Notify"
@@ -71,7 +74,7 @@
 
 ##############################################
 ## ApplicationAssedAssignedNotify
-if "application_assed_inprogress_notify" in NOTIFICATIONS:
+if "application:assed:inprogress:notify" in NOTIFICATIONS:
     aain_application = ApplicationFixtureFactory.make_application_source()
     aain_application["admin"]["editor"] = USER
     aain_application["bibjson"]["title"] = "Application Assed In Progress Notify"
@@ -85,7 +88,7 @@
 
 ##############################################
 ## ApplicationEditorCompletedNotify
-if "application_editor_completed_notify" in NOTIFICATIONS:
+if "application:editor:completed:notify" in NOTIFICATIONS:
     def editor_group_mock_pull(editor_group_id):
         return EditorGroup(**{
             "editor": USER
@@ -109,7 +112,7 @@ def editor_group_mock_pull(editor_group_id):
 
 ##############################################
 ## ApplicationEditorGroupAssignedNotify
-if "application_editor_group_assigned_notify" in NOTIFICATIONS:
+if "application:editor_group:assigned:notify" in NOTIFICATIONS:
     def editor_group_mock_pull(key, value):
         return EditorGroup(**{
             "editor": USER
@@ -133,7 +136,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## ApplicationEditorInprogressNotify
-if "application_editor_inprogress_notify" in NOTIFICATIONS:
+if "application:editor:inprogress:notify" in NOTIFICATIONS:
     def editor_group_mock_pull(editor_group_id):
         return EditorGroup(**{
             "editor": USER
@@ -157,7 +160,7 @@ def editor_group_mock_pull(editor_group_id):
 
 ##############################################
 ## ApplicationManedReadyNotify
-if "application_maned_ready_notify" in NOTIFICATIONS:
+if "application:maned:ready:notify" in NOTIFICATIONS:
     def editor_group_mock_pull(key, value):
         return EditorGroup(**{
             "maned": USER
@@ -181,7 +184,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## ApplicationPublisherAcceptedNotify
-if "application_publisher_accepted_notify" in NOTIFICATIONS:
+if "application:publisher:accepted:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Application Publisher Accepted Notify"
@@ -195,7 +198,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## ApplicationPublisherAssignedNotify
-if "application_publisher_assigned_notify" in NOTIFICATIONS:
+if "application:publisher:assigned:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Application Publisher Assigned Notify"
@@ -209,7 +212,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## ApplicationPublisherCreatedNotify
-if "application_publisher_created_notify" in NOTIFICATIONS:
+if "application:publisher:created:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Application Publisher Created Notify"
@@ -223,7 +226,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## ApplicationPublisherInprogressNotify
-if "application_publisher_inprogress_notify" in NOTIFICATIONS:
+if "application:publisher:inprogress:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Application Publisher In Progress Notify"
@@ -237,7 +240,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## ApplicationPublisherQuickRejectNotify
-if "application_publisher_quickreject_notify" in NOTIFICATIONS:
+if "application:publisher:quickreject:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Application Publisher Quick Reject Notify"
@@ -251,7 +254,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## ApplicationPublisherQuickRejectNotify
-if "application_publisher_revision_notify" in NOTIFICATIONS:
+if "application:publisher:revision:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Application Publisher Revision Notify"
@@ -267,7 +270,7 @@ def editor_group_mock_pull(key, value):
 ## BGJobFinishedNotify
 if "bg_job_finished_notify" in NOTIFICATIONS:
     job = models.BackgroundJob(**{
-        "id": "bg_job_finished_notify",
+        "id": "bg:job_finished:notify",
         "user": USER,
         "action": "bg_job_finished_notify",
         "status": "complete"
@@ -281,7 +284,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## JournalAssedAssignedNotify
-if "journal_assed_assigned_notify" in NOTIFICATIONS:
+if "journal:assed:assigned:notify" in NOTIFICATIONS:
     journal = JournalFixtureFactory.make_journal_source(in_doaj=True)
     journal["admin"]["editor"] = USER
     journal["bibjson"]["title"] = "Journal Assed Assigned Notify"
@@ -295,7 +298,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## JournalEditorGroupAssignedNotify
-if "journal_editor_group_assigned_notify" in NOTIFICATIONS:
+if "journal:editor_group:assigned:notify" in NOTIFICATIONS:
     def editor_group_mock_pull(key, value):
         return EditorGroup(**{
             "editor": USER
@@ -320,7 +323,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## UpdateRequestPublisherAcceptedNotify
-if "update_request_publisher_accepted_notify" in NOTIFICATIONS:
+if "update_request:publisher:accepted:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Update Request Publisher Accepted Notify"
@@ -334,7 +337,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## UpdateRequestPublisherAssignedNotify
-if "update_request_publisher_assigned_notify" in NOTIFICATIONS:
+if "update_request:publisher:assigned:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Update Request Publisher Assigned Notify"
@@ -348,7 +351,7 @@ def editor_group_mock_pull(key, value):
 
 ##############################################
 ## UpdateRequestPublisherRejectedNotify
-if "update_request_publisher_rejected_notify" in NOTIFICATIONS:
+if "update_request:publisher:rejected:notify" in NOTIFICATIONS:
     application = ApplicationFixtureFactory.make_application_source()
     application["admin"]["owner"] = USER
     application["bibjson"]["title"] = "Update Request Publisher Rejected Notify"
diff --git a/portality/bll/services/events.py b/portality/bll/services/events.py
index aa7a937f1..6ca7677e8 100644
--- a/portality/bll/services/events.py
+++ b/portality/bll/services/events.py
@@ -26,32 +26,32 @@
 
 
 class EventsService(object):
-    # disabled events - to enable move the event to EVENT_CONSUMENRS array
+    # disabled events - to enable move the event to EVENT_CONSUMERS array
     DISABLED_EVENTS = [
-        ApplicationPublisherRevisionNotify
+        ApplicationPublisherAssignedNotify, # https://github.com/DOAJ/doajPM/issues/3974
+        ApplicationPublisherInprogressNotify, # https://github.com/DOAJ/doajPM/issues/3974
+        ApplicationPublisherRevisionNotify,
+        JournalEditorGroupAssignedNotify, # https://github.com/DOAJ/doajPM/issues/3974
+        JournalAssedAssignedNotify, # https://github.com/DOAJ/doajPM/issues/3974
+        UpdateRequestPublisherAssignedNotify, # https://github.com/DOAJ/doajPM/issues/3974
     ]
     EVENT_CONSUMERS = [
-        ApplicationPublisherQuickRejectNotify,
         AccountCreatedEmail,
         AccountPasswordResetEmail,
-        ApplicationAssedInprogressNotify,
         ApplicationAssedAssignedNotify,
+        ApplicationAssedInprogressNotify,
         ApplicationEditorCompletedNotify,
-        ApplicationEditorInProgressNotify,
         ApplicationEditorGroupAssignedNotify,
+        ApplicationEditorInProgressNotify,
         ApplicationManedReadyNotify,
-        ApplicationPublisherCreatedNotify,
-        ApplicationPublisherInprogressNotify,
         ApplicationPublisherAcceptedNotify,
-        ApplicationPublisherAssignedNotify,
+        ApplicationPublisherCreatedNotify,
+        ApplicationPublisherQuickRejectNotify,
         BGJobFinishedNotify,
-        JournalAssedAssignedNotify,
-        JournalEditorGroupAssignedNotify,
+        JournalDiscontinuingSoonNotify,
         UpdateRequestPublisherAcceptedNotify,
-        UpdateRequestPublisherAssignedNotify,
         UpdateRequestPublisherRejectedNotify,
-        UpdateRequestPublisherSubmittedNotify,
-        JournalDiscontinuingSoonNotify,
+        UpdateRequestPublisherSubmittedNotify
     ]
 
     def __init__(self):
diff --git a/portality/forms/application_processors.py b/portality/forms/application_processors.py
index a7f027144..6afe65491 100644
--- a/portality/forms/application_processors.py
+++ b/portality/forms/application_processors.py
@@ -456,11 +456,6 @@ def finalise(self, account, save_target=True, email_alert=True):
                 #     self.add_alert("Problem sending email to associate editor - probably address is invalid")
                 #     app.logger.exception("Email to associate failed.")
 
-            # If this is the first time this application has been assigned to an editor, notify the publisher.
-            old_ed = self.source.editor
-            if (old_ed is None or old_ed == '') and self.target.editor is not None:
-                self.add_alert(Messages.SENT_PUBLISHER_ASSIGNED_EMAIL)
-
             # Inform editor and associate editor if this application was 'ready' or 'completed', but has been changed to 'in progress'
             if (self.source.application_status == constants.APPLICATION_STATUS_READY or self.source.application_status == constants.APPLICATION_STATUS_COMPLETED) and self.target.application_status == constants.APPLICATION_STATUS_IN_PROGRESS:
                 # First, the editor
@@ -586,11 +581,6 @@ def finalise(self):
             #     self.add_alert("Problem sending email to associate editor - probably address is invalid")
             #     app.logger.exception('Error sending associate assigned email')
 
-        # If this is the first time this application has been assigned to an editor, notify the publisher.
-        old_ed = self.source.editor
-        if (old_ed is None or old_ed == '') and self.target.editor is not None:
-            self.add_alert(Messages.SENT_PUBLISHER_ASSIGNED_EMAIL)
-
         # Email the assigned associate if the application was reverted from 'completed' to 'in progress' (failed review)
         if self.source.application_status == constants.APPLICATION_STATUS_COMPLETED and self.target.application_status == constants.APPLICATION_STATUS_IN_PROGRESS:
             if self.target.editor:
diff --git a/portality/ui/messages.py b/portality/ui/messages.py
index 5702b45aa..1487a2ca7 100644
--- a/portality/ui/messages.py
+++ b/portality/ui/messages.py
@@ -29,8 +29,6 @@ class Messages(object):
     SENT_JOURNAL_CONTACT_ACCEPTED_UPDATE_REQUEST_EMAIL = """Sent email to journal contact '{email}' to tell that an update to their journal was accepted."""
     SENT_JOURNAL_CONTACT_IN_PROGRESS_EMAIL = """An email has been sent to the Journal Contact alerting them that you are working on their application."""
     SENT_JOURNAL_CONTACT_ASSIGNED_EMAIL = """An email has been sent to the Journal Contact alerting them that an editor has been assigned to their application."""
-    SENT_PUBLISHER_IN_PROGRESS_EMAIL = """An email has been sent to the Owner alerting them that you are working on their application."""
-    SENT_PUBLISHER_ASSIGNED_EMAIL = """A notification has been sent to the Owner alerting them that an editor has been assigned to their application."""
 
     NOT_SENT_ACCEPTED_APPLICATION_EMAIL = """Did not send notification to '{user}' to tell them that their journal was accepted.  Email may be disabled, or there is a problem with the email address."""
     NOT_SENT_REJECTED_APPLICATION_EMAILS = """Did not send email to user '{user}' or application suggester to tell them that their journal was rejected  Email may be disabled, or there is a problem with the email address."""
@@ -40,8 +38,6 @@ class Messages(object):
     NOT_SENT_JOURNAL_CONTACT_ACCEPTED_APPLICATION_EMAIL = """Did not send email to '{email}' to tell them that their application/update request was accepted. Email may be disabled, or there is a problem with the email address"""
     NOT_SENT_JOURNAL_CONTACT_IN_PROGRESS_EMAIL = """An email could not be sent to the Journal Contact alerting them that you are working on their application. Email may be disabled, or there is a problem with the email address"""
     NOT_SENT_JOURNAL_CONTACT_ASSIGNED_EMAIL = """An email could not be sent to the Journal Contact alerting them that an editor has been assigned to their application. Email may be disabled, or there is a problem with the email address"""
-    NOT_SENT_PUBLISHER_IN_PROGRESS_EMAIL = """An email could not be sent to the Owner alerting them that you are working on their application. Email may be disabled, or there is a problem with the email address. """
-    NOT_SENT_PUBLISHER_ASSIGNED_EMAIL = """An email could not be sent to the Owner alerting them that an editor has been assigned to their application. Email may be disabled, or there is a problem with the email address"""
 
     IN_PROGRESS_NOT_SENT_EMAIL_DISABLED = """Did not send email to Owner or Journal Contact about the status change, as publisher emails are disabled."""
 

From cc8738acbc91f276e7c6f16559aa7224200c2f23 Mon Sep 17 00:00:00 2001
From: Dom Mitchell <dommitchell@users.noreply.github.com>
Date: Tue, 12 Nov 2024 13:48:59 +0100
Subject: [PATCH 65/70] Added l 91-155

---
 cms/pages/about/index.md | 78 +++++++++++++++++++++++++++++++++-------
 1 file changed, 66 insertions(+), 12 deletions(-)

diff --git a/cms/pages/about/index.md b/cms/pages/about/index.md
index f677f83f1..49dbeabd8 100644
--- a/cms/pages/about/index.md
+++ b/cms/pages/about/index.md
@@ -38,17 +38,6 @@ The work done by the [DOAJ Team](/about/team/) is supported by over 100 voluntar
 
 DOAJ also has a global network of Ambassadors who promote open access and best practice in their regions. Our Ambassadors are bound by an agreement and must declare all conflicts of interest.
 
-## Zero tolerance policy
-
-We expect the members of our Team, our volunteers and our Ambassadors to always be treated courteously and with respect for the work they are doing. They should not be subject to abusive behaviour in any form, which includes:
-
-- rude, disrespectful and offensive behaviour, including derogatory remarks or anything which amounts to verbal or emotional abuse in person or in communications
-- sexually inappropriate comments or behaviour
-- racist and discriminatory abuse
-- threats of physical violence
-- aggressive and violent behaviour
-
-
 ## Partnerships and collaborations
 
 DOAJ partners with many organisations. The nature of the partnerships varies and may include membership, contracts for work, exchanges of information or services, initiative signatories, or access to information resources that assist DOAJ with our application review process.
@@ -95,7 +84,72 @@ Established in 2015, Think. Check. Submit. was developed with the support of an
 
 Think. Check. Submit. provides tools and practical resources to educate researchers, promote integrity, and build trust in credible research and publications. Separate checklists for [journals](https://thinkchecksubmit.org/journals/) and [books](https://thinkchecksubmit.org/books-and-chapters/) are available to guide researchers through the key criteria for selecting where to publish their research and are also invaluable to scholarly communications professionals who are advising researchers about these issues. Both checklists are also available in a growing number of languages.
 
-
 ## Diversity
 
 DOAJ believes in the power and value of diversity in scholarly communications. DOAJ is an [adopting organisation](https://c4disc.org/about/adopting-organizations/) of the [C4DISC Joint Statement of Principles](https://c4disc.org/joint-statement-of-principles/). Adopting the Joint Statement of Principles demonstrates our support for improving diversity and inclusion in our industry.
+
+## Code of conduct and zero tolerance policy
+
+At DOAJ, we foster a welcoming and inclusive community for everyone. Our code of conduct and zero tolerance policy help us create an environment where you feel comfortable and valued. 
+
+### What this policy covers
+
+Our policy applies to all kinds of communication within our community or the spaces we create. They cover interactions among our team, volunteers, ambassadors, and individuals involved in events, activities and projects initiated or organised by us. This includes:
+
+- Communication via email or social media
+- Participation in webinars and events organised by us
+- Communication with our team members, volunteers and ambassadors at other events
+
+### Who this policy includes
+
+We welcome everyone, no matter your gender identity, sexual orientation, abilities, neurodiversity, appearance, body size, ethnicity, nationality, race, age, religion (or lack of it), social identity, or any other protected characteristic. Our community is wonderfully diverse and filled with passionate individuals who care deeply about their work. While our collaboration can sometimes spark lively discussions, we ask everyone to treat one another with consideration, kindness and respect.
+
+### Code of conduct
+
+We encourage members of our community who engage with or for DOAJ to:
+
+1. **Be kind and patient.** We want all members of our community to engage and be heard. We are a diverse community with many languages and many needs. Remember that people may use translation services or screen readers. They may be communicating in a second language.
+2. **Be respectful and open.** Our community is open scholarship, open research, open access, and open science. We encourage you to have open discussions. No matter the content, we ask you to communicate in a professional manner, be open to hearing others' points of view, and exchange information, experiences and knowledge openly.
+3. **Create a positive environment.** We want you to feel comfortable when engaging with us and our community. Learning and exploring different aspects of open scholarship should be fun! We encourage you to interact positively and share laughs, but avoid making demeaning or exclusionary jokes.
+4. **Speak up!** We encourage and support you in addressing unwelcome behaviour as long as you feel comfortable doing so. Otherwise, follow our process below to report issues.
+
+### Zero tolerance
+
+We have zero tolerance for:
+
+- Rude, disrespectful and offensive behaviour, including derogatory remarks or anything that amounts to verbal or emotional abuse
+- Sexually inappropriate comments or behaviour
+- Racist and discriminatory abuse
+- Threats of physical violence
+- Aggressive and violent behaviour
+- Doxxing (publishing private information about any of members of the DOAJ team, ambassadors or volunteers)
+- Stalking
+
+Or any other inappropriate behaviours that breach our code of conduct.
+
+### Reporting and resolving issues
+
+#### Reporting issues
+
+If you have concerns about someone’s behaviour, please contact the Operations Manager, Lene (lene@doaj.org). Lene will:
+
+- refer the incident to our Executive Team (who will decide on further action)
+- follow up with all those involved and provide you with an update
+
+#### Resolving issues
+
+In cases where our zero tolerance policy has not been breached, we will try to resolve the conflict, for example, through a mediated conversation involving a third party or separate conversations with everyone involved. Our aim is to solve issues respectfully and to improve relationships and collaborations. 
+
+We also encourage you to let us know about incidents that were successfully resolved without our involvement. This helps us ensure we have the correct policies in place.
+
+#### Breaches to our zero tolerance policy
+
+We expect you to treat members of our team, volunteers and ambassadors with kindness and respect for the work they are doing. If someone breaches our zero tolerance policy, we may take action such as:
+
+- Stopping all communication with you, including emails or any appeals and complaints
+- Excluding a publisher or journal from the DOAJ
+- Banning an individual from future DOAJ events
+- Blocking or reporting an individual or a profile on social media
+- Removing an individual from an event or preventing them from attending future events
+- Reporting incidents involving our team, volunteers or ambassadors at external events or withdrawing from an event
+- Making a statement on our blog

From 0b0eea6d149a4b52b431a2f6ae938b9963bb4f55 Mon Sep 17 00:00:00 2001
From: Richard Jones <richard@cottagelabs.com>
Date: Tue, 12 Nov 2024 13:19:09 +0000
Subject: [PATCH 66/70] Update form email sending expectations

---
 .../test_application_processor_emails.py      | 107 ++----------------
 1 file changed, 12 insertions(+), 95 deletions(-)

diff --git a/doajtest/unit/application_processors/test_application_processor_emails.py b/doajtest/unit/application_processors/test_application_processor_emails.py
index cc106942e..228d09405 100644
--- a/doajtest/unit/application_processors/test_application_processor_emails.py
+++ b/doajtest/unit/application_processors/test_application_processor_emails.py
@@ -354,16 +354,7 @@ def test_01_maned_review_emails(self):
                                         re.DOTALL)
         assert bool(assEd_email_matched), info_stream_contents.strip('\x00')
 
-        publisher_template = re.escape(templates.EMAIL_NOTIFICATION)
-        publisher_to = re.escape(owner.email)
-        publisher_subject = re.escape('Directory of Open Access Journals - Your application ({}) has been assigned to an editor for review'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        publisher_email_matched = re.search(email_log_regex % (publisher_template, publisher_to, publisher_subject),
-                                            info_stream_contents,
-                                            re.DOTALL)
-
-        assert bool(publisher_email_matched)
-        assert len(re.findall(email_count_string, info_stream_contents)) == 2
+        assert len(re.findall(email_count_string, info_stream_contents)) == 1
 
         # Clear the stream for the next part
         self.info_stream.truncate(0)
@@ -553,16 +544,7 @@ def test_02_ed_review_emails(self):
                                         info_stream_contents,
                                         re.DOTALL)
         assert bool(assEd_email_matched), info_stream_contents.strip('\x00')
-
-        publisher_template = templates.EMAIL_NOTIFICATION
-        publisher_to = re.escape(owner.email)
-        publisher_subject = re.escape('Your update request ({}) has been assigned to an editor for review'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        publisher_email_matched = re.search(email_log_regex % (publisher_template, publisher_to, publisher_subject),
-                                            info_stream_contents,
-                                            re.DOTALL)
-        assert bool(publisher_email_matched)
-        assert len(re.findall(email_count_string, info_stream_contents)) == 2
+        assert len(re.findall(email_count_string, info_stream_contents)) == 1
 
         # Clear the stream for the next part
         self.info_stream.truncate(0)
@@ -665,17 +647,8 @@ def test_03_assoc_ed_review_emails(self):
         processor.finalise()
         info_stream_contents = self.info_stream.getvalue()
 
-        # We expect one email to be sent here:
-        #   * to the publisher, notifying that an editor is viewing their application
-        publisher_template = re.escape(templates.EMAIL_NOTIFICATION)
-        publisher_to = re.escape(owner.email)
-        publisher_subject = re.escape('Directory of Open Access Journals - Your submission ({}) is under review'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        publisher_email_matched = re.search(email_log_regex % (publisher_template, publisher_to, publisher_subject),
-                                            info_stream_contents,
-                                            re.DOTALL)
-        assert bool(publisher_email_matched)
-        assert len(re.findall(email_count_string, info_stream_contents)) == 1
+        # We expect no emails
+        assert len(re.findall(email_count_string, info_stream_contents)) == 0
 
         # Clear the stream for the next part
         self.info_stream.truncate(0)
@@ -936,17 +909,7 @@ def test_01_maned_review_emails(self):
                                         info_stream_contents,
                                         re.DOTALL)
         assert bool(assEd_email_matched), info_stream_contents.strip('\x00')
-
-        publisher_template = templates.EMAIL_NOTIFICATION
-        publisher_to = re.escape(owner.email)
-        publisher_subject = re.escape('Your update request ({}) has been assigned to an editor for review'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        publisher_email_matched = re.search(email_log_regex % (publisher_template, publisher_to, publisher_subject),
-                                            info_stream_contents,
-                                            re.DOTALL)
-
-        assert bool(publisher_email_matched)
-        assert len(re.findall(email_count_string, info_stream_contents)) == 2
+        assert len(re.findall(email_count_string, info_stream_contents)) == 1
 
         # Clear the stream for the next part
         self.info_stream.truncate(0)
@@ -1125,16 +1088,7 @@ def test_02_ed_review_emails(self):
                                         info_stream_contents,
                                         re.DOTALL)
         assert bool(assEd_email_matched), info_stream_contents.strip('\x00')
-
-        publisher_template = templates.EMAIL_NOTIFICATION
-        publisher_to = re.escape(owner.email)
-        publisher_subject = re.escape('Your update request ({}) has been assigned to an editor for review'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        publisher_email_matched = re.search(email_log_regex % (publisher_template, publisher_to, publisher_subject),
-                                            info_stream_contents,
-                                            re.DOTALL)
-        assert bool(publisher_email_matched)
-        assert len(re.findall(email_count_string, info_stream_contents)) == 2
+        assert len(re.findall(email_count_string, info_stream_contents)) == 1
 
         # Clear the stream for the next part
         self.info_stream.truncate(0)
@@ -1240,17 +1194,8 @@ def test_03_assoc_ed_review_emails(self):
         processor.finalise()
         info_stream_contents = self.info_stream.getvalue()
 
-        # We expect one email to be sent here:
-        #   * to the publisher, notifying that an editor is viewing their application
-        publisher_template = re.escape(templates.EMAIL_NOTIFICATION)
-        publisher_to = re.escape(owner.email)
-        publisher_subject = re.escape('Your submission ({}) is under review'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        publisher_email_matched = re.search(email_log_regex % (publisher_template, publisher_to, publisher_subject),
-                                            info_stream_contents,
-                                            re.DOTALL)
-        assert bool(publisher_email_matched)
-        assert len(re.findall(email_count_string, info_stream_contents)) == 1
+        # We expect no email to be sent
+        assert len(re.findall(email_count_string, info_stream_contents)) == 0
 
         # Clear the stream for the next part
         self.info_stream.truncate(0)
@@ -1328,27 +1273,8 @@ def test_01_maned_review_emails(self):
         # check the associate was changed
         assert processor.target.editor == "associate_3"
 
-        # We expect 2 emails to be sent:
-        #   * to the editor of the assigned group,
-        #   * to the AssEd who's been assigned,
-        editor_template = re.escape(templates.EMAIL_NOTIFICATION)
-        editor_to = re.escape('eddie@example.com')
-        editor_subject = re.escape('Directory of Open Access Journals - New journal ({}) assigned to your group'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        editor_email_matched = re.search(email_log_regex % (editor_template, editor_to, editor_subject),
-                                         info_stream_contents,
-                                         re.DOTALL)
-        assert bool(editor_email_matched)
-
-        assEd_template = re.escape(templates.EMAIL_NOTIFICATION)
-        assEd_to = re.escape(models.Account.pull('associate_3').email)
-        assEd_subject = re.escape('Directory of Open Access Journals - New journal ({}) assigned to you'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        assEd_email_matched = re.search(email_log_regex % (assEd_template, assEd_to, assEd_subject),
-                                        info_stream_contents,
-                                        re.DOTALL)
-        assert bool(assEd_email_matched), info_stream_contents.strip('\x00')
-        assert len(re.findall(email_count_string, info_stream_contents)) == 2
+        # We expect no emails to be sent
+        assert len(re.findall(email_count_string, info_stream_contents)) == 0
         ctx.pop()
 
     def test_02_ed_review_emails(self):
@@ -1369,16 +1295,7 @@ def test_02_ed_review_emails(self):
         # check the associate was changed
         assert processor.target.editor == "associate_2"
 
-        # We expect 1 email to be sent:
-        #   * to the AssEd who's been assigned
-        assEd_template = re.escape(templates.EMAIL_NOTIFICATION)
-        assEd_to = re.escape(models.Account.pull('associate_2').email)
-        assEd_subject = re.escape('Directory of Open Access Journals - New journal ({}) assigned to you'.format(', '.join(issn for issn in processor.source.bibjson().issns())))
-
-        assEd_email_matched = re.search(email_log_regex % (assEd_template, assEd_to, assEd_subject),
-                                        info_stream_contents,
-                                        re.DOTALL)
-        assert bool(assEd_email_matched), info_stream_contents.strip('\x00')
-        assert len(re.findall(email_count_string, info_stream_contents)) == 1
+        # We no email to be sent
+        assert len(re.findall(email_count_string, info_stream_contents)) == 0
 
         ctx.pop()

From 5ea6b51becb976ebd6ebedabd868af0c7f0afcb5 Mon Sep 17 00:00:00 2001
From: Steven Eardley <steve@eardley.xyz>
Date: Tue, 12 Nov 2024 16:02:34 +0000
Subject: [PATCH 67/70] correct the edges version in develop

---
 portality/static/vendor/edges | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/portality/static/vendor/edges b/portality/static/vendor/edges
index 9639b871a..990f42201 160000
--- a/portality/static/vendor/edges
+++ b/portality/static/vendor/edges
@@ -1 +1 @@
-Subproject commit 9639b871acb1f6590ee78e236f2c9333479c9fe8
+Subproject commit 990f4220163a3e18880f0bdc3ad5c80d234d22dd

From bc293fff5b8dc0b5da6c65e41087e5511dfbf0ff Mon Sep 17 00:00:00 2001
From: Steven Eardley <steve@eardley.xyz>
Date: Tue, 12 Nov 2024 16:06:25 +0000
Subject: [PATCH 68/70] version bump for release

---
 portality/settings.py | 2 +-
 setup.py              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/portality/settings.py b/portality/settings.py
index 213961706..dbeddc63e 100644
--- a/portality/settings.py
+++ b/portality/settings.py
@@ -9,7 +9,7 @@
 # Application Version information
 # ~~->API:Feature~~
 
-DOAJ_VERSION = "7.0.1"
+DOAJ_VERSION = "7.0.2"
 API_VERSION = "4.0.0"
 
 ######################################
diff --git a/setup.py b/setup.py
index ddc27e972..bfebfeed0 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 
 setup(
     name='doaj',
-    version='7.0.1',
+    version='7.0.2',
     packages=find_packages(),
     install_requires=[
         "awscli==1.20.50",

From 82434b8d3970a34634224b002b8ef3a08540f6ad Mon Sep 17 00:00:00 2001
From: Steven Eardley <steve@eardley.xyz>
Date: Thu, 14 Nov 2024 12:09:02 +0000
Subject: [PATCH 69/70] Static pages release version bump

---
 portality/settings.py | 2 +-
 setup.py              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/portality/settings.py b/portality/settings.py
index dbeddc63e..e6bd2ae57 100644
--- a/portality/settings.py
+++ b/portality/settings.py
@@ -9,7 +9,7 @@
 # Application Version information
 # ~~->API:Feature~~
 
-DOAJ_VERSION = "7.0.2"
+DOAJ_VERSION = "7.0.3"
 API_VERSION = "4.0.0"
 
 ######################################
diff --git a/setup.py b/setup.py
index bfebfeed0..953c5fe42 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 
 setup(
     name='doaj',
-    version='7.0.2',
+    version='7.0.3',
     packages=find_packages(),
     install_requires=[
         "awscli==1.20.50",

From e89b5fedaa42c727a6492ddbfaf592e70b5aab67 Mon Sep 17 00:00:00 2001
From: Steven Eardley <steve@eardley.xyz>
Date: Fri, 15 Nov 2024 14:32:24 +0000
Subject: [PATCH 70/70] expand logrotate documentation

---
 deploy/logrotate/README.md | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/deploy/logrotate/README.md b/deploy/logrotate/README.md
index ec163fd12..2ef592cca 100644
--- a/deploy/logrotate/README.md
+++ b/deploy/logrotate/README.md
@@ -30,4 +30,32 @@ Log uploads can be checked on the S3 bucket with the following command:
 
 ```
 aws --profile doaj-nginx-logs s3 ls s3://doaj-nginx-logs
-```
\ No newline at end of file
+```
+
+You can try a test run of `logrotate`:
+
+```
+sudo /usr/sbin/logrotate /etc/logrotate.conf
+```
+
+Or sync the files by directly running:
+
+```
+HOSTNAME=`hostname` sudo aws --profile doaj-nginx-logs s3 sync /var/log/nginx/ s3://doaj-nginx-logs/$HOSTNAME/ --exclude "*" --include "doaj.*.gz"
+```
+
+
+### Troubleshooting:
+
+```
+error: nginx:1 duplicate log entry for /var/log/nginx/doaj.access.log
+error: found error in file nginx, skipping
+```
+
+Check whether you've disabled the default nginx configuration
+
+```
+The config profile (doaj-nginx-logs) could not be found
+```
+
+Check that root user has access to the AWS credentials and symlink if required.
\ No newline at end of file