From 34efbd1bdbbc0f1399cd9fa426eeee54961f5977 Mon Sep 17 00:00:00 2001 From: John Connor Date: Thu, 15 Feb 2024 12:09:47 -0600 Subject: [PATCH] patched --- .../commands/iiif_generate_manifests.py | 12 +++- ...ibrary_page_transkribus_pageid_and_more.py | 18 +++++ ..._remove_page_iiif_manifest_url_and_more.py | 68 +++++++++++++++++++ ...urce_bib_source_thumbnail_transcription.py | 34 ---------- .../0008_source_manifest_content.py | 18 ----- ..._zotero_url_alter_source_title_and_more.py | 36 ---------- .../migrations/0010_alter_source_title.py | 18 ----- ..._source_has_published_manifest_and_more.py | 23 ------- ...uid_source_zotero_web_page_url_and_more.py | 19 ------ ..._remove_page_iiif_manifest_url_and_more.py | 21 ------ 10 files changed, 96 insertions(+), 171 deletions(-) create mode 100644 api/document/migrations/0007_remove_page_iiif_manifest_url_and_more.py delete mode 100644 api/document/migrations/0007_source_bib_source_thumbnail_transcription.py delete mode 100644 api/document/migrations/0008_source_manifest_content.py delete mode 100644 api/document/migrations/0009_sourcetype_source_zotero_url_alter_source_title_and_more.py delete mode 100644 api/document/migrations/0010_alter_source_title.py delete mode 100644 api/document/migrations/0011_alter_source_has_published_manifest_and_more.py delete mode 100644 api/document/migrations/0012_source_uid_source_zotero_web_page_url_and_more.py delete mode 100644 api/document/migrations/0013_remove_page_iiif_manifest_url_and_more.py diff --git a/api/document/management/commands/iiif_generate_manifests.py b/api/document/management/commands/iiif_generate_manifests.py index e80e8ea..aafe104 100644 --- a/api/document/management/commands/iiif_generate_manifests.py +++ b/api/document/management/commands/iiif_generate_manifests.py @@ -64,6 +64,8 @@ def handle(self, *args, **options): manifest_sources=Source.objects.all().filter(~Q(page_connections__page=None)) + print(manifest_sources) + #screen out sources that lack either pages sources = Source.objects \ .prefetch_related('page_connections') \ @@ -73,7 +75,8 @@ def handle(self, *args, **options): .prefetch_related('source_enslaver_connections__enslaver') \ .prefetch_related('source_enslaved_connections__enslaved') \ .filter( - ~Q(page_connections__page=None) and ~Q(manifest_content=None) + ~Q(page_connections__page=None) +# and ~Q(manifest_content=None) ) print(f"Found {sources.count()} sources with page images.") @@ -90,7 +93,9 @@ def handle(self, *args, **options): generated_count=0 for source in sources: with transaction.atomic(): + print("--->",source.title) content = source.manifest_content +# print("------->",content) #then do a final pass to ensure that we don't have "pages" without images #some of those did sneak in during the process of indexing transkribus against the library collections @@ -220,7 +225,10 @@ def handle(self, *args, **options): # Append entity connections to metadata. doc_links = {} - metadata = list(content['metadata']) + if content is not None: + metadata = list(content['metadata']) + else: + metadata=[] #voyage ids source_voyages=source.source_voyage_connections.all() diff --git a/api/document/migrations/0006_page_is_british_library_page_transkribus_pageid_and_more.py b/api/document/migrations/0006_page_is_british_library_page_transkribus_pageid_and_more.py index 1e89d24..ddc9839 100644 --- a/api/document/migrations/0006_page_is_british_library_page_transkribus_pageid_and_more.py +++ b/api/document/migrations/0006_page_is_british_library_page_transkribus_pageid_and_more.py @@ -1,6 +1,7 @@ # Generated by Django 4.2.1 on 2023-11-20 17:10 from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): @@ -10,6 +11,13 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='SourceType', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ], + ), migrations.AddField( model_name='page', name='is_british_library', @@ -40,4 +48,14 @@ class Migration(migrations.Migration): name='zotero_grouplibrary_name', field=models.CharField(default='sv-docs', max_length=255), ), + migrations.CreateModel( + name='Transcription', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('language_code', models.CharField(max_length=20)), + ('text', models.TextField()), + ('is_translation', models.BooleanField()), + ('page', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcriptions', to='document.page')), + ], + ) ] diff --git a/api/document/migrations/0007_remove_page_iiif_manifest_url_and_more.py b/api/document/migrations/0007_remove_page_iiif_manifest_url_and_more.py new file mode 100644 index 0000000..6577731 --- /dev/null +++ b/api/document/migrations/0007_remove_page_iiif_manifest_url_and_more.py @@ -0,0 +1,68 @@ +# Generated by Django 4.2.1 on 2024-02-15 17:35 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('document', '0006_page_is_british_library_page_transkribus_pageid_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='page', + name='iiif_manifest_url', + ), + migrations.RemoveField( + model_name='page', + name='transcription', + ), + migrations.AddField( + model_name='source', + name='bib', + field=models.TextField(blank=True, help_text='Formatted bibliography for the Document', null=True), + ), + migrations.AddField( + model_name='source', + name='manifest_content', + field=models.JSONField(blank=True, help_text='DCTerms imported from Zotero -- NOT the full manifest', null=True), + ), + migrations.AddField( + model_name='source', + name='source_type', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='document.sourcetype'), + ), + migrations.AddField( + model_name='source', + name='thumbnail', + field=models.TextField(blank=True, help_text='URL for a thumbnail of the Document', null=True), + ), + migrations.AddField( + model_name='source', + name='zotero_url', + field=models.URLField(blank=True, max_length=400, null=True), + ), + migrations.AlterField( + model_name='docsparsedate', + name='year', + field=models.IntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(2050)]), + ), + migrations.AlterField( + model_name='source', + name='has_published_manifest', + field=models.BooleanField(default=False, verbose_name='Is there a published manifest?'), + ), + migrations.AlterField( + model_name='source', + name='item_url', + field=models.URLField(blank=True, max_length=400, null=True), + ), + migrations.AlterField( + model_name='source', + name='title', + field=models.CharField(max_length=1000, verbose_name='Title'), + ), + ] diff --git a/api/document/migrations/0007_source_bib_source_thumbnail_transcription.py b/api/document/migrations/0007_source_bib_source_thumbnail_transcription.py deleted file mode 100644 index ac82c21..0000000 --- a/api/document/migrations/0007_source_bib_source_thumbnail_transcription.py +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by Django 4.2.1 on 2024-01-12 17:03 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('document', '0006_page_is_british_library_page_transkribus_pageid_and_more'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='bib', - field=models.TextField(help_text='Formatted bibliography for the Document', null=True), - ), - migrations.AddField( - model_name='source', - name='thumbnail', - field=models.TextField(help_text='URL for a thumbnail of the Document', null=True), - ), - migrations.CreateModel( - name='Transcription', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('language_code', models.CharField(max_length=20)), - ('text', models.TextField()), - ('is_translation', models.BooleanField()), - ('page', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcriptions', to='document.page')), - ], - ), - ] diff --git a/api/document/migrations/0008_source_manifest_content.py b/api/document/migrations/0008_source_manifest_content.py deleted file mode 100644 index 5e7ce8b..0000000 --- a/api/document/migrations/0008_source_manifest_content.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.1 on 2024-01-16 15:43 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('document', '0007_source_bib_source_thumbnail_transcription'), - ] - - operations = [ - migrations.AddField( - model_name='source', - name='manifest_content', - field=models.JSONField(null=True), - ), - ] diff --git a/api/document/migrations/0009_sourcetype_source_zotero_url_alter_source_title_and_more.py b/api/document/migrations/0009_sourcetype_source_zotero_url_alter_source_title_and_more.py deleted file mode 100644 index 1d361c7..0000000 --- a/api/document/migrations/0009_sourcetype_source_zotero_url_alter_source_title_and_more.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 4.2.1 on 2024-01-16 16:03 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('document', '0008_source_manifest_content'), - ] - - operations = [ - migrations.CreateModel( - name='SourceType', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255, unique=True)), - ], - ), - migrations.AddField( - model_name='source', - name='zotero_url', - field=models.URLField(max_length=400, null=True), - ), - migrations.AlterField( - model_name='source', - name='title', - field=models.CharField(max_length=500, verbose_name='Title'), - ), - migrations.AddField( - model_name='source', - name='source_type', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='document.sourcetype'), - ), - ] diff --git a/api/document/migrations/0010_alter_source_title.py b/api/document/migrations/0010_alter_source_title.py deleted file mode 100644 index 60b2728..0000000 --- a/api/document/migrations/0010_alter_source_title.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.1 on 2024-01-16 17:52 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('document', '0009_sourcetype_source_zotero_url_alter_source_title_and_more'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='title', - field=models.CharField(max_length=1000, verbose_name='Title'), - ), - ] diff --git a/api/document/migrations/0011_alter_source_has_published_manifest_and_more.py b/api/document/migrations/0011_alter_source_has_published_manifest_and_more.py deleted file mode 100644 index 0b22d93..0000000 --- a/api/document/migrations/0011_alter_source_has_published_manifest_and_more.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 4.2.1 on 2024-01-18 21:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('document', '0010_alter_source_title'), - ] - - operations = [ - migrations.AlterField( - model_name='source', - name='has_published_manifest', - field=models.BooleanField(default=False, verbose_name='Is there a published manifest?'), - ), - migrations.AlterField( - model_name='source', - name='manifest_content', - field=models.JSONField(help_text='DCTerms imported from Zotero -- NOT the full manifest', null=True), - ), - ] diff --git a/api/document/migrations/0012_source_uid_source_zotero_web_page_url_and_more.py b/api/document/migrations/0012_source_uid_source_zotero_web_page_url_and_more.py deleted file mode 100644 index b849b2c..0000000 --- a/api/document/migrations/0012_source_uid_source_zotero_web_page_url_and_more.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 4.2.1 on 2024-01-24 19:37 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('document', '0011_alter_source_has_published_manifest_and_more'), - ] - - operations = [ - migrations.AlterField( - model_name='docsparsedate', - name='year', - field=models.IntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(2050)]), - ), - ] diff --git a/api/document/migrations/0013_remove_page_iiif_manifest_url_and_more.py b/api/document/migrations/0013_remove_page_iiif_manifest_url_and_more.py deleted file mode 100644 index e308603..0000000 --- a/api/document/migrations/0013_remove_page_iiif_manifest_url_and_more.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 4.2.1 on 2024-01-25 22:04 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('document', '0012_source_uid_source_zotero_web_page_url_and_more'), - ] - - operations = [ - migrations.RemoveField( - model_name='page', - name='iiif_manifest_url', - ), - migrations.RemoveField( - model_name='page', - name='transcription', - ), - ]