Skip to content

Commit

Permalink
Merge pull request #101 from acdh-oeaw/99-upgrade-all-packages
Browse files Browse the repository at this point in the history
99 upgrade all packages
  • Loading branch information
csae8092 authored Sep 19, 2024
2 parents 56afa09 + 1bcd7a4 commit 535e7e7
Show file tree
Hide file tree
Showing 109 changed files with 1,857 additions and 1,068 deletions.
4 changes: 3 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,6 @@ env
db.sqlite3
.env
env.default
*.sql
*.sql
.git
/venv
5 changes: 3 additions & 2 deletions .flake8
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[flake8]
ignore = D203 W504
ignore = D203 W504 W503
max-line-length = 120
exclude =
*/migrations,
Expand All @@ -11,4 +11,5 @@ exclude =
dist
env
appcreator
webpage
webpage
venv
10 changes: 5 additions & 5 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,26 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3

- name: Log in to the Container registry
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v3
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

- name: Build and push Docker image
uses: docker/build-push-action@v3
uses: docker/build-push-action@v6
with:
context: .
push: true
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ jobs:
name: Lint
steps:
- name: Check out source repository
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Set up Python environment
uses: actions/setup-python@v1
uses: actions/setup-python@v5
with:
python-version: "3.8"
python-version: "3.12"
- name: flake8 Lint
uses: py-actions/flake8@v1
uses: py-actions/flake8@v2
18 changes: 8 additions & 10 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
POSTGRES_DB: github_actions
services:
postgres:
image: postgres:13
image: postgres:16
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
Expand All @@ -19,17 +19,15 @@ jobs:
# needed because the postgres container does not provide a healthcheck
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.8
uses: actions/setup-python@v1
- uses: actions/checkout@v4
- name: Set up Python 12
uses: actions/setup-python@v5
with:
python-version: 3.8
- uses: actions/cache@v2
python-version: "3.12"
- uses: actions/cache@v4
with:
path: ${{ env.pythonLocation }}
key: ${{ env.pythonLocation }}-${{ hashFiles('requirements_dev.txt') }}
- name: GDAL
run: sudo apt-get install postgresql-common libpq-dev python3-gdal
- name: Install dependencies
run: |
pip install --upgrade --upgrade-strategy eager -r requirements_dev.txt
Expand All @@ -40,8 +38,8 @@ jobs:
- name: Create Coverage Report
run: coverage xml
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v1
uses: codecov/codecov-action@v4
with:
token: ${{secrets.CODECOV_TOKEN}}
file: ./coverage.xml
fail_ci_if_error: true
fail_ci_if_error: false
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,5 @@ key.pub
bin/
pg_hba.conf
lib64
*sql
*sql
/venv
35 changes: 9 additions & 26 deletions archiv/dal_views.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,7 @@
# generated by appcreator
from django.db.models import Q
from dal import autocomplete
from . models import (
Event,
Work,
Person,
Place,
Institution,
Wab
)
from .models import Event, Work, Person, Place, Institution, Wab


class PersonAC(autocomplete.Select2QuerySetView):
Expand All @@ -17,9 +10,9 @@ def get_queryset(self):

if self.q:
qs = qs.filter(
Q(title__icontains=self.q) |
Q(gnd_gnd_id=self.q) |
Q(gnd_pref_name=self.q)
Q(title__icontains=self.q)
| Q(gnd_gnd_id=self.q)
| Q(gnd_pref_name=self.q)
)
return qs

Expand All @@ -29,9 +22,7 @@ def get_queryset(self):
qs = Place.objects.all()

if self.q:
qs = qs.filter(
Q(title__icontains=self.q)
)
qs = qs.filter(Q(title__icontains=self.q))
return qs


Expand All @@ -40,9 +31,7 @@ def get_queryset(self):
qs = Wab.objects.all()

if self.q:
qs = qs.filter(
Q(title__icontains=self.q)
)
qs = qs.filter(Q(title__icontains=self.q))
return qs


Expand All @@ -51,9 +40,7 @@ def get_queryset(self):
qs = Institution.objects.all()

if self.q:
qs = qs.filter(
Q(title__icontains=self.q)
)
qs = qs.filter(Q(title__icontains=self.q))
return qs


Expand All @@ -62,10 +49,7 @@ def get_queryset(self):
qs = Event.objects.all()

if self.q:
qs = qs.filter(
Q(id__icontains=self.q) |
Q(full_text__icontains=self.q)
)
qs = qs.filter(Q(id__icontains=self.q) | Q(full_text__icontains=self.q))
return qs


Expand All @@ -75,7 +59,6 @@ def get_queryset(self):

if self.q:
qs = qs.filter(
Q(order_code__icontains=self.q) |
Q(full_quote__icontains=self.q)
Q(order_code__icontains=self.q) | Q(full_quote__icontains=self.q)
)
return qs
10 changes: 5 additions & 5 deletions archiv/data_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@ def event_calendar_data(request):
data = []
for x in Event.objects.exclude(not_before__isnull=True):
item = {
'id': x.date_written,
'name': f"{x}",
'startDate': f"{x.not_before}",
'linkId': x.get_absolute_url()
"id": x.date_written,
"name": f"{x}",
"startDate": f"{x.not_before}",
"linkId": x.get_absolute_url(),
}
data.append(item)
var_data = f"var calendarData = {json.dumps(data, ensure_ascii=False)}"
return HttpResponse(var_data, content_type="text/javascript")


class EventCalendarView(TemplateView):
template_name = 'archiv/event_calendar.html'
template_name = "archiv/event_calendar.html"
8 changes: 4 additions & 4 deletions archiv/import_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@


def field_mapping(some_class):
""" returns a dictionary mapping model field names to lookukp values
:param some_class: Any django model class with extra field properties
:return: A dict mapping model field names to lookukp values
"""returns a dictionary mapping model field names to lookukp values
:param some_class: Any django model class with extra field properties
:return: A dict mapping model field names to lookukp values
"""
field_mapping_dict = {}
for x in some_class._meta.get_fields():
try:
field_mapping_dict[(x.extra['data_lookup']).strip()] = x.name
field_mapping_dict[(x.extra["data_lookup"]).strip()] = x.name
except: # noqa: E722
pass
return field_mapping_dict
Expand Down
44 changes: 24 additions & 20 deletions archiv/management/commands/bruckner_entities.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,43 +12,47 @@
NS_LIST = {
"tei": "http://www.tei-c.org/ns/1.0",
"mei": "http://www.music-encoding.org/ns/mei",
"xml": "http://www.w3.org/XML/1998/namespace"
"xml": "http://www.w3.org/XML/1998/namespace",
}


class Command(BaseCommand):
help = 'Enriches Persons with bruckner_entities_xml data'
help = "Enriches Persons with bruckner_entities_xml data"

def handle(self, *args, **kwargs):
headers = {
"PRIVATE-TOKEN": settings.GITLAB_TOKEN
}
headers = {"PRIVATE-TOKEN": settings.GITLAB_TOKEN}
r = requests.get(settings.BRUCKNER_PERSON_URL, headers=headers)
content = r.content.decode('utf-8')
content = content.replace('http://www.music-encoding.org/ns/mei', 'http://www.tei-c.org/ns/1.0')
content = r.content.decode("utf-8")
content = content.replace(
"http://www.music-encoding.org/ns/mei", "http://www.tei-c.org/ns/1.0"
)
doc = TeiReader(content)
person_entries = doc.any_xpath('.//tei:persName[@xml:id]')
person_entries = doc.any_xpath(".//tei:persName[@xml:id]")
for x in tqdm(person_entries, total=len(person_entries)):
item = {}
item['xml_id'] = x.attrib['{http://www.w3.org/XML/1998/namespace}id']
item["xml_id"] = x.attrib["{http://www.w3.org/XML/1998/namespace}id"]
try:
item['last_name'] = x.xpath('.//tei:famName/text()', namespaces=NS_LIST)[0]
item["last_name"] = x.xpath(
".//tei:famName/text()", namespaces=NS_LIST
)[0]
except IndexError:
item['last_name'] = ''
item["last_name"] = ""
try:
item['first_name'] = x.xpath('.//tei:foreName/text()', namespaces=NS_LIST)[0]
item["first_name"] = x.xpath(
".//tei:foreName/text()", namespaces=NS_LIST
)[0]
except IndexError:
item['first_name'] = ''
item['label'] = f"{item['first_name'].strip()} {item['last_name'].strip()}"
item['xml_node'] = ET.tostring(x)
item["first_name"] = ""
item["label"] = f"{item['first_name'].strip()} {item['last_name'].strip()}"
item["xml_node"] = ET.tostring(x)
try:
pers = Person.objects.get(title=item['label'])
pers = Person.objects.get(title=item["label"])
except ObjectDoesNotExist:
pers, _ = Person.objects.get_or_create(
bruckner_entity_id=item['xml_id']
bruckner_entity_id=item["xml_id"]
)
pers.title = item['label']
pers.title = item["label"]
pers.bruckner_entity = True
pers.bruckner_entity_id = item['xml_id']
pers.bruckner_entity_xml = item['xml_node'].decode('utf-8')
pers.bruckner_entity_id = item["xml_id"]
pers.bruckner_entity_xml = item["xml_node"].decode("utf-8")
pers.save()
12 changes: 6 additions & 6 deletions archiv/management/commands/event_wab.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@


class Command(BaseCommand):
help = 'Links WAB to Events based on Wab-Titles'
help = "Links WAB to Events based on Wab-Titles"

def handle(self, *args, **kwargs):
sheet_id = settings.WAB_SEARCH
Expand All @@ -20,10 +20,10 @@ def handle(self, *args, **kwargs):
try:
wab = Wab.objects.get(wab_id=wab_id)
except ObjectDoesNotExist:
data.append([wab_id, row['match_str'], 0])
data.append([wab_id, row["match_str"], 0])
continue
matches = Event.objects.filter(main_text__contains=row['match_str'])
matches = Event.objects.filter(main_text__contains=row["match_str"])
wab.rvn_wab_mentioned_in.add(*matches)
data.append([wab_id, row['match_str'], matches.count()])
out_df = pd.DataFrame(data, columns=['wab', 'search_str', 'matches'])
out_df.to_csv('./media/wab_matching.csv', index=False)
data.append([wab_id, row["match_str"], matches.count()])
out_df = pd.DataFrame(data, columns=["wab", "search_str", "matches"])
out_df.to_csv("./media/wab_matching.csv", index=False)
4 changes: 2 additions & 2 deletions archiv/management/commands/fix_encoding.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@


class Command(BaseCommand):
help = 'Fix wrong encoding of latin chars'
help = "Fix wrong encoding of latin chars"

def handle(self, *args, **kwargs):
fields = Event.search_field_names()
Expand All @@ -17,7 +17,7 @@ def handle(self, *args, **kwargs):
old = getattr(x, f)
if old is not None:
for i, row in df.iterrows():
old = old.replace(row['from'], row['to'])
old = old.replace(row["from"], row["to"])
setattr(x, f, old)
x.save()
print("done")
8 changes: 4 additions & 4 deletions archiv/management/commands/fix_stichworte.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,19 @@


class Command(BaseCommand):
help = 'Imports text_Stichworte Data'
help = "Imports text_Stichworte Data"

def handle(self, *args, **kwargs):
sheet_id = settings.LEGACY_DB_SHEET_ID
self.stdout.write(f"fetching data from {sheet_id}")
df = gsheet_to_df(sheet_id)
df = df.dropna(subset=['text_Stichworte', 'input_Sortiercode'])
df = df.dropna(subset=["text_Stichworte", "input_Sortiercode"])
for i, row in tqdm(df.iterrows(), total=len(df)):
try:
event = Event.objects.get(id=row['input_Sortiercode'])
event = Event.objects.get(id=row["input_Sortiercode"])
except ObjectDoesNotExist:
continue
stichwort = row['text_Stichworte']
stichwort = row["text_Stichworte"]
event.key_word = stichwort
event.save()
self.stdout.write("done")
Loading

0 comments on commit 535e7e7

Please sign in to comment.