diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b07c3177bd..e50b7a716f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -70,6 +70,7 @@ jobs: - weko-schema-ui - weko-search-ui - weko-sitemap + - weko-swordserver - weko-theme - weko-user-profiles - weko-workflow diff --git a/docker-compose.yml b/docker-compose.yml index bbe4e643ab..f222a69341 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -212,6 +212,7 @@ services: - INVENIO_DB_POOL_CLASS=NullPool - GOOGLE_TRACKING_ID_SYSTEM=UA-86504114-1 - GOOGLE_TRACKING_ID_USER= + - TMPDIR=/var/tmp volumes: - weko3_data:/var/tmp - static_data:/home/invenio/.virtualenvs/invenio/var/instance/static diff --git a/docker-compose2.yml b/docker-compose2.yml index bbe4e643ab..f222a69341 100644 --- a/docker-compose2.yml +++ b/docker-compose2.yml @@ -212,6 +212,7 @@ services: - INVENIO_DB_POOL_CLASS=NullPool - GOOGLE_TRACKING_ID_SYSTEM=UA-86504114-1 - GOOGLE_TRACKING_ID_USER= + - TMPDIR=/var/tmp volumes: - weko3_data:/var/tmp - static_data:/home/invenio/.virtualenvs/invenio/var/instance/static diff --git a/freeze.txt b/freeze.txt index cfa19c2b7f..604b0667ce 100644 --- a/freeze.txt +++ b/freeze.txt @@ -176,7 +176,7 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 +Pillow==8.1.2 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 diff --git a/modules/invenio-accounts/requirements.txt b/modules/invenio-accounts/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-accounts/requirements.txt +++ b/modules/invenio-accounts/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-accounts/requirements2.txt b/modules/invenio-accounts/requirements2.txt index 58e3e1e5c1..40e717e79a 100644 --- a/modules/invenio-accounts/requirements2.txt +++ b/modules/invenio-accounts/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,7 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +202,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-accounts/tox.result.gz b/modules/invenio-accounts/tox.result.gz index ded6a78727..d432659e1d 100644 Binary files a/modules/invenio-accounts/tox.result.gz and b/modules/invenio-accounts/tox.result.gz differ diff --git a/modules/invenio-communities/requirements.txt b/modules/invenio-communities/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-communities/requirements.txt +++ b/modules/invenio-communities/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-communities/requirements2.txt b/modules/invenio-communities/requirements2.txt index f87df00774..9913675b07 100644 --- a/modules/invenio-communities/requirements2.txt +++ b/modules/invenio-communities/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,7 +195,7 @@ pyparsing==3.1.0 pyPEG2==2.15.2 pytest==4.2.0 pytest-mock -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-communities/tox.result.gz b/modules/invenio-communities/tox.result.gz index ac3bc42e23..68586e3f0a 100644 Binary files a/modules/invenio-communities/tox.result.gz and b/modules/invenio-communities/tox.result.gz differ diff --git a/modules/invenio-db/coverage.xml b/modules/invenio-db/coverage.xml index 349ca4c504..03ad7d433a 100644 --- a/modules/invenio-db/coverage.xml +++ b/modules/invenio-db/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-db/requirements.txt b/modules/invenio-db/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-db/requirements.txt +++ b/modules/invenio-db/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-db/requirements2.txt b/modules/invenio-db/requirements2.txt index f87df00774..9913675b07 100644 --- a/modules/invenio-db/requirements2.txt +++ b/modules/invenio-db/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,7 +195,7 @@ pyparsing==3.1.0 pyPEG2==2.15.2 pytest==4.2.0 pytest-mock -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-db/tox.result.gz b/modules/invenio-db/tox.result.gz index c97f02f8ac..739b7c955d 100644 Binary files a/modules/invenio-db/tox.result.gz and b/modules/invenio-db/tox.result.gz differ diff --git a/modules/invenio-deposit/coverage.xml b/modules/invenio-deposit/coverage.xml index b535127594..18da44172d 100644 --- a/modules/invenio-deposit/coverage.xml +++ b/modules/invenio-deposit/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-deposit/requirements.txt b/modules/invenio-deposit/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-deposit/requirements.txt +++ b/modules/invenio-deposit/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-deposit/requirements2.txt b/modules/invenio-deposit/requirements2.txt index 58e3e1e5c1..9913675b07 100644 --- a/modules/invenio-deposit/requirements2.txt +++ b/modules/invenio-deposit/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-deposit/tox.result.gz b/modules/invenio-deposit/tox.result.gz index 90c4cdd171..90f4b77afc 100644 Binary files a/modules/invenio-deposit/tox.result.gz and b/modules/invenio-deposit/tox.result.gz differ diff --git a/modules/invenio-files-rest/coverage.xml b/modules/invenio-files-rest/coverage.xml index 485720774a..c65b07e7d0 100644 --- a/modules/invenio-files-rest/coverage.xml +++ b/modules/invenio-files-rest/coverage.xml @@ -1,12 +1,12 @@ - + /code/modules/invenio-files-rest/invenio_files_rest - + @@ -433,7 +433,7 @@ - + @@ -483,8 +483,8 @@ - - + + @@ -578,7 +578,7 @@ - + @@ -877,39 +877,39 @@ - - - - + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - + diff --git a/modules/invenio-files-rest/requirements.txt b/modules/invenio-files-rest/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-files-rest/requirements.txt +++ b/modules/invenio-files-rest/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-files-rest/requirements2.txt b/modules/invenio-files-rest/requirements2.txt index 58e3e1e5c1..9913675b07 100644 --- a/modules/invenio-files-rest/requirements2.txt +++ b/modules/invenio-files-rest/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-files-rest/tox.result.gz b/modules/invenio-files-rest/tox.result.gz index f43901159a..551557b47a 100644 Binary files a/modules/invenio-files-rest/tox.result.gz and b/modules/invenio-files-rest/tox.result.gz differ diff --git a/modules/invenio-iiif/coverage.xml b/modules/invenio-iiif/coverage.xml index cecf02b2fd..604f43fb38 100644 --- a/modules/invenio-iiif/coverage.xml +++ b/modules/invenio-iiif/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-iiif/requirements.txt b/modules/invenio-iiif/requirements.txt index bf7652b1ec..d05965c836 100644 --- a/modules/invenio-iiif/requirements.txt +++ b/modules/invenio-iiif/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-iiif/requirements2.txt b/modules/invenio-iiif/requirements2.txt index f1ec27eb49..41c12fbcfe 100644 --- a/modules/invenio-iiif/requirements2.txt +++ b/modules/invenio-iiif/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,7 +195,7 @@ pyparsing==3.1.0 pyPEG2==2.15.2 pytest>=4.2.0 pytest-mock==3.6.1 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-iiif/tox.result.gz b/modules/invenio-iiif/tox.result.gz index 1b1c94309a..999463033a 100644 Binary files a/modules/invenio-iiif/tox.result.gz and b/modules/invenio-iiif/tox.result.gz differ diff --git a/modules/invenio-indexer/requirements.txt b/modules/invenio-indexer/requirements.txt index 958c77f054..97f8290c6b 100644 --- a/modules/invenio-indexer/requirements.txt +++ b/modules/invenio-indexer/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -116,7 +117,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -124,6 +125,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -163,7 +165,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-indexer/requirements2.txt b/modules/invenio-indexer/requirements2.txt index 58e3e1e5c1..9913675b07 100644 --- a/modules/invenio-indexer/requirements2.txt +++ b/modules/invenio-indexer/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-indexer/tox.result.gz b/modules/invenio-indexer/tox.result.gz index 06bf401986..20bd0f8dfa 100644 Binary files a/modules/invenio-indexer/tox.result.gz and b/modules/invenio-indexer/tox.result.gz differ diff --git a/modules/invenio-mail/coverage.xml b/modules/invenio-mail/coverage.xml index 5dcd014a96..538fe9ed22 100644 --- a/modules/invenio-mail/coverage.xml +++ b/modules/invenio-mail/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-mail/requirements.txt b/modules/invenio-mail/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-mail/requirements.txt +++ b/modules/invenio-mail/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-mail/requirements2.txt b/modules/invenio-mail/requirements2.txt index f1ec27eb49..41c12fbcfe 100644 --- a/modules/invenio-mail/requirements2.txt +++ b/modules/invenio-mail/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,7 +195,7 @@ pyparsing==3.1.0 pyPEG2==2.15.2 pytest>=4.2.0 pytest-mock==3.6.1 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-mail/tox.result.gz b/modules/invenio-mail/tox.result.gz index 5c9a0a788c..a5bcf80fa7 100644 Binary files a/modules/invenio-mail/tox.result.gz and b/modules/invenio-mail/tox.result.gz differ diff --git a/modules/invenio-oaiharvester/coverage.xml b/modules/invenio-oaiharvester/coverage.xml index 4cc74d81a5..c0d840ebb6 100644 --- a/modules/invenio-oaiharvester/coverage.xml +++ b/modules/invenio-oaiharvester/coverage.xml @@ -1,12 +1,12 @@ - + /code/modules/invenio-oaiharvester/invenio_oaiharvester - + @@ -19,7 +19,7 @@ - + @@ -74,30 +74,30 @@ - - - - - + + + + + - - - + + + - - - - - - - - - - + + + + + + + + + + - + @@ -117,19 +117,19 @@ - - - - - - - - - + + + + + + + + + - + @@ -151,7 +151,7 @@ - + @@ -168,14 +168,14 @@ - + - + - + @@ -189,7 +189,7 @@ - + @@ -204,7 +204,7 @@ - + @@ -219,33 +219,33 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -378,7 +378,7 @@ - + @@ -809,8 +809,8 @@ - - + + @@ -1236,7 +1236,7 @@ - + @@ -1323,8 +1323,8 @@ - - + + @@ -1332,7 +1332,7 @@ - + @@ -1340,30 +1340,30 @@ - - + + - - - + + + - + - + - + - - - - - + + + + + @@ -1379,10 +1379,10 @@ - - - - + + + + @@ -1394,7 +1394,7 @@ - + @@ -1423,90 +1423,90 @@ - - + + - - - - + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - + + + + + + @@ -1516,27 +1516,27 @@ - - - - - - - - + + + + + + + + - + - - - - - - - - + + + + + + + + diff --git a/modules/invenio-oaiharvester/requirements.txt b/modules/invenio-oaiharvester/requirements.txt index ec3deeb16e..a87373db8f 100644 --- a/modules/invenio-oaiharvester/requirements.txt +++ b/modules/invenio-oaiharvester/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-oaiharvester/requirements2.txt b/modules/invenio-oaiharvester/requirements2.txt index 6fc0e8fd84..1aa00ff2df 100644 --- a/modules/invenio-oaiharvester/requirements2.txt +++ b/modules/invenio-oaiharvester/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +204,12 @@ redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 responses==0.10.3 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-oaiharvester/tox.result.gz b/modules/invenio-oaiharvester/tox.result.gz index ab1537cee0..e929f671f5 100644 Binary files a/modules/invenio-oaiharvester/tox.result.gz and b/modules/invenio-oaiharvester/tox.result.gz differ diff --git a/modules/invenio-oaiserver/requirements.txt b/modules/invenio-oaiserver/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-oaiserver/requirements.txt +++ b/modules/invenio-oaiserver/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-oaiserver/requirements2.txt b/modules/invenio-oaiserver/requirements2.txt index f1ec27eb49..41c12fbcfe 100644 --- a/modules/invenio-oaiserver/requirements2.txt +++ b/modules/invenio-oaiserver/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,7 +195,7 @@ pyparsing==3.1.0 pyPEG2==2.15.2 pytest>=4.2.0 pytest-mock==3.6.1 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-oaiserver/tox.result.gz b/modules/invenio-oaiserver/tox.result.gz index d469f19b1b..050d0edf56 100644 Binary files a/modules/invenio-oaiserver/tox.result.gz and b/modules/invenio-oaiserver/tox.result.gz differ diff --git a/modules/invenio-oauth2server/coverage.xml b/modules/invenio-oauth2server/coverage.xml index 64679745d5..1196a65b2a 100644 --- a/modules/invenio-oauth2server/coverage.xml +++ b/modules/invenio-oauth2server/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-oauth2server/requirements.txt b/modules/invenio-oauth2server/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-oauth2server/requirements.txt +++ b/modules/invenio-oauth2server/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-oauth2server/requirements2.txt b/modules/invenio-oauth2server/requirements2.txt index 58e3e1e5c1..9913675b07 100644 --- a/modules/invenio-oauth2server/requirements2.txt +++ b/modules/invenio-oauth2server/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-oauth2server/tox.result.gz b/modules/invenio-oauth2server/tox.result.gz index 5aa9398ff5..2e22014181 100644 Binary files a/modules/invenio-oauth2server/tox.result.gz and b/modules/invenio-oauth2server/tox.result.gz differ diff --git a/modules/invenio-previewer/coverage.xml b/modules/invenio-previewer/coverage.xml index 8fb214e65f..c8c49ab3f1 100644 --- a/modules/invenio-previewer/coverage.xml +++ b/modules/invenio-previewer/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-previewer/requirements.txt b/modules/invenio-previewer/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-previewer/requirements.txt +++ b/modules/invenio-previewer/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-previewer/requirements2.txt b/modules/invenio-previewer/requirements2.txt index 58e3e1e5c1..9913675b07 100644 --- a/modules/invenio-previewer/requirements2.txt +++ b/modules/invenio-previewer/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-previewer/tox.result.gz b/modules/invenio-previewer/tox.result.gz index d96a390b3f..9826bb0c17 100644 Binary files a/modules/invenio-previewer/tox.result.gz and b/modules/invenio-previewer/tox.result.gz differ diff --git a/modules/invenio-queues/coverage.xml b/modules/invenio-queues/coverage.xml index 664bd5106b..28a8fc6634 100644 --- a/modules/invenio-queues/coverage.xml +++ b/modules/invenio-queues/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-queues/requirements.txt b/modules/invenio-queues/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-queues/requirements.txt +++ b/modules/invenio-queues/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-queues/requirements2.txt b/modules/invenio-queues/requirements2.txt index 58e3e1e5c1..9913675b07 100644 --- a/modules/invenio-queues/requirements2.txt +++ b/modules/invenio-queues/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-queues/tox.result.gz b/modules/invenio-queues/tox.result.gz index 9977bc7d3f..7026b27587 100644 Binary files a/modules/invenio-queues/tox.result.gz and b/modules/invenio-queues/tox.result.gz differ diff --git a/modules/invenio-records-rest/coverage.xml b/modules/invenio-records-rest/coverage.xml index 6bf64d00da..92ef174804 100644 --- a/modules/invenio-records-rest/coverage.xml +++ b/modules/invenio-records-rest/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-records-rest/invenio_records_rest/views.py b/modules/invenio-records-rest/invenio_records_rest/views.py index ef6ac3cf10..8e7745f5d4 100644 --- a/modules/invenio-records-rest/invenio_records_rest/views.py +++ b/modules/invenio-records-rest/invenio_records_rest/views.py @@ -1146,6 +1146,7 @@ def put(self, pid, record, **kwargs): except BaseException as e: db.session.rollback() current_app.logger.error(traceback.format_exc()) + return self.make_response(None, None, 500) return self.make_response( pid, record, links_factory=self.links_factory) diff --git a/modules/invenio-records-rest/requirements.txt b/modules/invenio-records-rest/requirements.txt index 1c3309e482..a3e12953cc 100644 --- a/modules/invenio-records-rest/requirements.txt +++ b/modules/invenio-records-rest/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -116,7 +117,7 @@ PyJWT==1.5.3 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 pytest-mock==3.6.1 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -124,6 +125,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -163,7 +165,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-records-rest/requirements2.txt b/modules/invenio-records-rest/requirements2.txt index 6e78d0c928..00ce98bd94 100644 --- a/modules/invenio-records-rest/requirements2.txt +++ b/modules/invenio-records-rest/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -178,7 +179,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -197,7 +197,7 @@ pyPEG2==2.15.2 pytest==4.2.0 pytest-mock pytest-invenio -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -205,10 +205,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-records-rest/tox.result.gz b/modules/invenio-records-rest/tox.result.gz index 75de7dffcb..29a84389b4 100644 Binary files a/modules/invenio-records-rest/tox.result.gz and b/modules/invenio-records-rest/tox.result.gz differ diff --git a/modules/invenio-records/coverage.xml b/modules/invenio-records/coverage.xml index 6f1d3670fe..74667cdc48 100644 --- a/modules/invenio-records/coverage.xml +++ b/modules/invenio-records/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-records/requirements.txt b/modules/invenio-records/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-records/requirements.txt +++ b/modules/invenio-records/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-records/requirements2.txt b/modules/invenio-records/requirements2.txt index 58e3e1e5c1..9913675b07 100644 --- a/modules/invenio-records/requirements2.txt +++ b/modules/invenio-records/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-records/tox.result.gz b/modules/invenio-records/tox.result.gz index f80fbdb3f0..bb40143928 100644 Binary files a/modules/invenio-records/tox.result.gz and b/modules/invenio-records/tox.result.gz differ diff --git a/modules/invenio-resourcesyncclient/coverage.xml b/modules/invenio-resourcesyncclient/coverage.xml index 599339ce96..bff4e60bfc 100644 --- a/modules/invenio-resourcesyncclient/coverage.xml +++ b/modules/invenio-resourcesyncclient/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-resourcesyncclient/requirements.txt b/modules/invenio-resourcesyncclient/requirements.txt index d4f77aa639..16a5433190 100644 --- a/modules/invenio-resourcesyncclient/requirements.txt +++ b/modules/invenio-resourcesyncclient/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-resourcesyncclient/requirements2.txt b/modules/invenio-resourcesyncclient/requirements2.txt index 58e3e1e5c1..9913675b07 100644 --- a/modules/invenio-resourcesyncclient/requirements2.txt +++ b/modules/invenio-resourcesyncclient/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -194,7 +194,8 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -202,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-resourcesyncclient/tox.result.gz b/modules/invenio-resourcesyncclient/tox.result.gz index 391f045aef..dfd52dc331 100644 Binary files a/modules/invenio-resourcesyncclient/tox.result.gz and b/modules/invenio-resourcesyncclient/tox.result.gz differ diff --git a/modules/invenio-resourcesyncserver/coverage.xml b/modules/invenio-resourcesyncserver/coverage.xml index ebe4f84645..98888637d4 100644 --- a/modules/invenio-resourcesyncserver/coverage.xml +++ b/modules/invenio-resourcesyncserver/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-resourcesyncserver/requirements.txt b/modules/invenio-resourcesyncserver/requirements.txt index d4f77aa639..16a5433190 100644 --- a/modules/invenio-resourcesyncserver/requirements.txt +++ b/modules/invenio-resourcesyncserver/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-resourcesyncserver/requirements2.txt b/modules/invenio-resourcesyncserver/requirements2.txt index f87df00774..9913675b07 100644 --- a/modules/invenio-resourcesyncserver/requirements2.txt +++ b/modules/invenio-resourcesyncserver/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,7 +195,7 @@ pyparsing==3.1.0 pyPEG2==2.15.2 pytest==4.2.0 pytest-mock -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-resourcesyncserver/tox.result.gz b/modules/invenio-resourcesyncserver/tox.result.gz index 4e203b7b39..e6c2838593 100644 Binary files a/modules/invenio-resourcesyncserver/tox.result.gz and b/modules/invenio-resourcesyncserver/tox.result.gz differ diff --git a/modules/invenio-s3/coverage.xml b/modules/invenio-s3/coverage.xml index 3049bde1e9..f789598844 100644 --- a/modules/invenio-s3/coverage.xml +++ b/modules/invenio-s3/coverage.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/invenio-s3/requirements.txt b/modules/invenio-s3/requirements.txt index 31e7139cce..8a5d255154 100644 --- a/modules/invenio-s3/requirements.txt +++ b/modules/invenio-s3/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -116,7 +117,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -124,6 +125,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -163,7 +165,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-s3/requirements2.txt b/modules/invenio-s3/requirements2.txt index 3ecd8ddb70..cac347c073 100644 --- a/modules/invenio-s3/requirements2.txt +++ b/modules/invenio-s3/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -178,7 +179,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,8 +195,9 @@ pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 pytest==4.2.0 +pytest-mock pytest-invenio==1.2.1 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -204,10 +205,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-s3/tests/conftest.py b/modules/invenio-s3/tests/conftest.py index 7169059d27..e912f13304 100644 --- a/modules/invenio-s3/tests/conftest.py +++ b/modules/invenio-s3/tests/conftest.py @@ -17,6 +17,7 @@ from flask import Flask, current_app from invenio_db import InvenioDB from invenio_db import db as db_ +from invenio_db.utils import drop_alembic_version_table from invenio_files_rest import InvenioFilesREST from invenio_files_rest.models import Location from moto import mock_s3 @@ -25,6 +26,7 @@ from invenio_s3 import InvenioS3, S3FSFileStorage + @pytest.fixture(scope='module') def app_config(app_config): """Customize application configuration.""" @@ -56,6 +58,45 @@ def factory(**config): return factory +@pytest.fixture(scope='module') +def base_app(): + """Flask application fixture.""" + app = Flask('testapp') + + app.config.update( + FILES_REST_STORAGE_FACTORY='invenio_s3:s3_storage_factory', + S3_ENDPOINT_URL=None, + S3_ACCCESS_KEY_ID= '', + S3_SECRECT_ACCESS_KEY = '', + SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', + 'postgresql+psycopg2://invenio:dbpass123@postgresql:5432/wekotest'), + SQLALCHEMY_TRACK_MODIFICATIONS = True, + TESTING = True + ) + + InvenioDB(app) + InvenioFilesREST(app) + InvenioS3(app) + + return app + +@pytest.yield_fixture(scope='module') +def app(base_app): + """Flask application fixture.""" + with base_app.app_context(): + yield base_app + + +@pytest.yield_fixture(scope='module') +def database(app): + """Get setup database.""" + if not database_exists(str(db_.engine.url)): + create_database(str(db_.engine.url)) + db_.create_all() + yield db_ + db_.session.remove() + db_.drop_all() + drop_alembic_version_table() @pytest.fixture(scope='module') def location_path(): diff --git a/modules/invenio-s3/tests/test_invenio_s3.py b/modules/invenio-s3/tests/test_invenio_s3.py index 55d78e4a16..f1e6f7a889 100644 --- a/modules/invenio-s3/tests/test_invenio_s3.py +++ b/modules/invenio-s3/tests/test_invenio_s3.py @@ -18,6 +18,7 @@ def test_version(): assert __version__ +# .tox/c1/bin/pytest --cov=invenio_s3 tests/test_invenio_s3.py::test_init -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-s3/.tox/c1/tmp def test_init(base_app, location, database): """Test extension initialization.""" assert 'invenio-s3' in base_app.extensions diff --git a/modules/invenio-s3/tox.result.gz b/modules/invenio-s3/tox.result.gz index cd93263905..4e5c1842f8 100644 Binary files a/modules/invenio-s3/tox.result.gz and b/modules/invenio-s3/tox.result.gz differ diff --git a/modules/invenio-stats/coverage.xml b/modules/invenio-stats/coverage.xml index cc1cabc435..fbeaf5547c 100644 --- a/modules/invenio-stats/coverage.xml +++ b/modules/invenio-stats/coverage.xml @@ -1,12 +1,12 @@ - + /code/modules/invenio-stats/invenio_stats - + @@ -404,19 +404,19 @@ - - - - - - - - - - - - + + + + + + + + + + + + @@ -795,7 +795,7 @@ - + @@ -1115,6 +1115,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/invenio-stats/invenio_stats/config.py b/modules/invenio-stats/invenio_stats/config.py index d802501c59..e93b83a014 100644 --- a/modules/invenio-stats/invenio_stats/config.py +++ b/modules/invenio-stats/invenio_stats/config.py @@ -143,7 +143,8 @@ 'get-file-download-per-site-license': {}, 'get-file-preview-per-site-license': {}, 'get-ranking-data': {}, - 'get-new-items-data': {} + 'get-new-items-data': {}, + 'item-file-download-aggs': {}, } STATS_PERMISSION_FACTORY = weko_permission_factory diff --git a/modules/invenio-stats/invenio_stats/contrib/registrations.py b/modules/invenio-stats/invenio_stats/contrib/registrations.py index f8e63c8e47..f36d61cb4e 100644 --- a/modules/invenio-stats/invenio_stats/contrib/registrations.py +++ b/modules/invenio-stats/invenio_stats/contrib/registrations.py @@ -19,7 +19,7 @@ from invenio_stats.processors import EventsIndexer, anonymize_user, \ flag_restricted, flag_robots from invenio_stats.queries import ESDateHistogramQuery, ESTermsQuery, \ - ESWekoFileStatsQuery, ESWekoTermsQuery, ESWekoRankingQuery + ESWekoFileStatsQuery, ESWekoTermsQuery, ESWekoRankingQuery, ESWekoFileRankingQuery from weko_schema_ui.models import PublishStatus @@ -911,5 +911,40 @@ def register_queries(): } } ) - ) + ), + dict( + query_name='item-file-download-aggs', + query_class=ESWekoFileRankingQuery, + query_config=dict( + index='{}-events-stats-file-download'.format(search_index_prefix), + doc_type='stats-file-download', + copy_fields=dict(), + metric_fields=dict( + download_ranking=('terms', 'file_key', {}) + ), + main_fields=['item_id'], + main_query={ + "query": { + "bool": { + "filter": [ + { + "term": { + "item_id": { + "value": "@item_id", + "boost": 1 + } + } + }, + { + "terms": { + "root_file_id": "@root_file_id_list" + } + } + ] + } + }, + "size": 0 + }, + ) + ), ] diff --git a/modules/invenio-stats/invenio_stats/queries.py b/modules/invenio-stats/invenio_stats/queries.py index b12a5b48c5..e6e01d5bf3 100644 --- a/modules/invenio-stats/invenio_stats/queries.py +++ b/modules/invenio-stats/invenio_stats/queries.py @@ -307,7 +307,7 @@ def build_buckets(agg, fields, bucket_result): """Build recursively result buckets.""" # Add metric results for current bucket for metric in self.metric_fields: - bucket_result[metric] = agg[metric]['value'] + bucket_result[metric] = agg[metric].get('value') if self.group_fields: temp_data = {} count = 0 @@ -597,4 +597,85 @@ def run(self, start_date=None, end_date=None, **kwargs): query_result = agg_query.execute().to_dict() - return query_result \ No newline at end of file + return query_result + + +class ESWekoFileRankingQuery(ESTermsQuery): + """Weko ES Query for File Download Ranking.""" + + def __init__(self, main_fields=None, main_query=None, *args, **kwargs): + """Constructor. + + :param main_fields: name of the timestamp field. + :param main_query: list of fields to copy from the top hit document + into the resulting aggregation. + """ + super(ESWekoFileRankingQuery, self).__init__(*args, **kwargs) + self.main_fields = main_fields or [] + self.main_query = main_query or {} + + def build_query(self, start_date, end_date, **kwargs): + """Build the elasticsearch query.""" + agg_query = Search( + using=self.client, + index=self.index, + doc_type=self.doc_type + )[0:0] + if self.main_query: + query_q = self.main_query + for _field in self.main_fields: + query_q = json.dumps(query_q).replace( + "@{}".format(_field), kwargs[_field]) + query_q = json.loads(query_q) + + if 'root_file_id_list' in kwargs.keys(): + query_q['query']['bool']['filter'][1]['terms']['root_file_id'] = kwargs['root_file_id_list'] + + agg_query.update_from_dict(query_q) + + if start_date or end_date: + time_range = {} + if start_date: + time_range['gte'] = start_date.isoformat() + if end_date: + time_range['lte'] = end_date.isoformat() + time_range['time_zone'] = current_app.config[ + 'STATS_WEKO_DEFAULT_TIMEZONE'] + agg_query = agg_query.filter( + 'range', + **{self.time_field: time_range}) + + for dst, (metric, field, opts) in self.metric_fields.items(): + agg_query.aggs.metric(dst, metric, field=field, **opts) + + return agg_query + + def process_query_result(self, query_result, start_date, end_date): + """Build the result using the query result.""" + def build_buckets(agg, fields, bucket_result): + """Build recursively result buckets.""" + # Add metric results for current bucket + for metric in self.metric_fields: + bucket_result[metric] = agg[metric] + return bucket_result + + # Add copy_fields + aggs = query_result['aggregations'] + result = dict( + start_date=start_date.isoformat() if start_date else None, + end_date=end_date.isoformat() if end_date else None, + ) + + return build_buckets(aggs, self.aggregated_fields, result) + + def run(self, start_date=None, end_date=None, **kwargs): + """Run the query.""" + start_date = self.extract_date(start_date) if start_date else None + end_date = self.extract_date(end_date) if end_date else None + self.validate_arguments(start_date, end_date, **kwargs) + + agg_query = self.build_query(start_date, end_date, **kwargs) + current_app.logger.debug(agg_query.to_dict()) + query_result = agg_query.execute().to_dict() + + return self.process_query_result(query_result, start_date, end_date) \ No newline at end of file diff --git a/modules/invenio-stats/requirements.txt b/modules/invenio-stats/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/invenio-stats/requirements.txt +++ b/modules/invenio-stats/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/invenio-stats/requirements2.txt b/modules/invenio-stats/requirements2.txt index cce3dcdfca..bf93f2420a 100644 --- a/modules/invenio-stats/requirements2.txt +++ b/modules/invenio-stats/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,7 +195,8 @@ pyparsing==3.1.0 pyPEG2==2.15.2 pytest==4.2.0 pytest-invenio==1.2.1 -python-dateutil==2.6.1 +pytest-mock +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +204,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/invenio-stats/tests/conftest.py b/modules/invenio-stats/tests/conftest.py index 8f7dd0c352..06e840e519 100644 --- a/modules/invenio-stats/tests/conftest.py +++ b/modules/invenio-stats/tests/conftest.py @@ -282,7 +282,8 @@ def base_app(instance_path, mock_gethostbyaddr): STATS_QUERIES=STATS_QUERIES, STATS_EVENTS=stats_events, STATS_AGGREGATIONS=STATS_AGGREGATIONS, - INDEXER_MQ_QUEUE = Queue("indexer", exchange=Exchange("indexer", type="direct"), routing_key="indexer",queue_arguments={"x-queue-type":"quorum"}) + INDEXER_MQ_QUEUE = Queue("indexer", exchange=Exchange("indexer", type="direct"), routing_key="indexer",queue_arguments={"x-queue-type":"quorum"}), + INDEXER_DEFAULT_INDEX="test-events-stats-file-download-0001" )) FlaskCeleryExt(app_) InvenioAccess(app_) @@ -1071,3 +1072,23 @@ def run(self, *args, **kwargs): """Sample response.""" return dict(bucket_id='test_bucket', value=100) + + +@pytest.fixture() +def esindex(app): + from invenio_search import current_search_client as client + index_name = app.config["INDEXER_DEFAULT_INDEX"] + alias_name = "test-events-stats-file-download" + + with open("tests/data/mappings/stats-file-download.json","r") as f: + mapping = json.load(f) + + with app.test_request_context(): + client.indices.create(index=index_name, body=mapping, ignore=[400]) + client.indices.put_alias(index=index_name, name=alias_name) + + yield client + + with app.test_request_context(): + client.indices.delete_alias(index=index_name, name=alias_name) + client.indices.delete(index=index_name, ignore=[400, 404]) diff --git a/modules/invenio-stats/tests/data/mappings/stats-file-download.json b/modules/invenio-stats/tests/data/mappings/stats-file-download.json new file mode 100644 index 0000000000..9c8e2788a3 --- /dev/null +++ b/modules/invenio-stats/tests/data/mappings/stats-file-download.json @@ -0,0 +1,184 @@ +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1, + "index.mapping.total_fields.limit": 50000, + "analysis": { + "tokenizer": { + "ja_tokenizer": { + "type": "kuromoji_tokenizer", + "mode": "search" + }, + "ngram_tokenizer": { + "type": "nGram", + "min_gram": 1, + "max_gram": 3, + "token_chars": [ + "letter", + "digit" + ] + } + }, + "char_filter": { + "weko_char_filter": { + "type": "mapping", + "mappings_path": "kui.txt" + } + }, + "analyzer": { + "default": { + "tokenizer": "ngram_tokenizer", + "filter": [ + "kuromoji_baseform", + "kuromoji_part_of_speech", + "cjk_width", + "stop", + "kuromoji_stemmer", + "lowercase" + ], + "char_filter": [ + "weko_char_filter" + ] + }, + "ngram_analyzer": { + "type": "custom", + "char_filter": [ + "weko_char_filter", + "html_strip" + ], + "tokenizer": "ngram_tokenizer", + "filter": [ + "cjk_width", + "lowercase" + ] + }, + "wk_analyzer": { + "type": "custom", + "char_filter": [ + "html_strip" + ], + "tokenizer": "standard", + "filter": [ + "standard", + "lowercase", + "stop", + "cjk_width" + ] + }, + "paths": { + "tokenizer": "path_hierarchy" + } + } + } + }, + "mappings": { + "stats-file-download": { + "dynamic": "false", + "dynamic_templates": [ + { + "date_fields": { + "match_mapping_type": "date", + "mapping": { + "format": "strict_date_hour_minute_second", + "type": "date" + } + } + } + ], + "date_detection": false, + "numeric_detection": false, + "properties": { + "accessrole": { + "type": "keyword" + }, + "billing_file_price": { + "type": "keyword" + }, + "bucket_id": { + "type": "keyword" + }, + "collection": { + "type": "keyword" + }, + "country": { + "type": "keyword" + }, + "cur_user_id": { + "type": "keyword" + }, + "file_id": { + "type": "keyword" + }, + "file_key": { + "type": "keyword" + }, + "hostname": { + "type": "keyword" + }, + "index_list": { + "type": "keyword" + }, + "is_billing_item": { + "type": "boolean" + }, + "is_restricted": { + "type": "boolean" + }, + "is_robot": { + "type": "boolean" + }, + "item_id": { + "type": "keyword" + }, + "item_title": { + "type": "keyword" + }, + "referrer": { + "type": "keyword" + }, + "remote_addr": { + "type": "keyword" + }, + "root_file_id": { + "type": "keyword" + }, + "site_license_flag": { + "type": "boolean" + }, + "site_license_name": { + "type": "keyword" + }, + "size": { + "type": "double" + }, + "timestamp": { + "type": "date", + "format": "strict_date_hour_minute_second" + }, + "unique_id": { + "type": "keyword" + }, + "unique_session_id": { + "type": "keyword" + }, + "user_group_list": { + "type": "nested", + "properties": { + "group_id": { + "type": "keyword" + }, + "group_name": { + "type": "keyword" + } + } + }, + "userrole": { + "type": "keyword" + }, + "visitor_id": { + "type": "keyword" + } + } + } + } +} \ No newline at end of file diff --git a/modules/invenio-stats/tests/data/test_events/event_download01.json b/modules/invenio-stats/tests/data/test_events/event_download01.json new file mode 100644 index 0000000000..2f0e270e2b --- /dev/null +++ b/modules/invenio-stats/tests/data/test_events/event_download01.json @@ -0,0 +1,28 @@ +{ + "timestamp": "2024-01-31T01:01:01", + "bucket_id": "6fd627e8-e7ba-4837-8f75-54286575efda", + "file_id": "root_file_id_01", + "root_file_id": "root_file_id_01", + "file_key": "test_file_01.txt", + "size": 100, + "referrer": null, + "accessrole": "open_access", + "userrole": "System Administrator", + "site_license_name": "", + "site_license_flag": false, + "index_list": "利用報告", + "cur_user_id": 1, + "item_id": "1", + "item_title": "test", + "remote_addr": "192.168.56.1", + "is_billing_item": false, + "billing_file_price": "", + "user_group_list": null, + "is_restricted": false, + "is_robot": false, + "country": null, + "visitor_id": "e80a50a34252bc7ac5d6ea1e2fbc34bbc3eb3d98f92e31749ee06ef1", + "unique_session_id": "db731d4ba06907cb6ddcc9950e636eee5b2205e71e98af9004e9db2f", + "unique_id": "be175de4-34bd-3042-982a-1344ad93a8f0", + "hostname": "None" +} \ No newline at end of file diff --git a/modules/invenio-stats/tests/data/test_events/event_download02.json b/modules/invenio-stats/tests/data/test_events/event_download02.json new file mode 100644 index 0000000000..254db324fd --- /dev/null +++ b/modules/invenio-stats/tests/data/test_events/event_download02.json @@ -0,0 +1,28 @@ +{ + "timestamp": "2023-12-31T23:59:59", + "bucket_id": "6fd627e8-e7ba-4837-8f75-54286575efda", + "file_id": "root_file_id_02", + "root_file_id": "root_file_id_02", + "file_key": "test_file_02.txt", + "size": 100, + "referrer": null, + "accessrole": "open_access", + "userrole": "System Administrator", + "site_license_name": "", + "site_license_flag": false, + "index_list": "利用報告", + "cur_user_id": 1, + "item_id": "1", + "item_title": "test", + "remote_addr": "192.168.56.1", + "is_billing_item": false, + "billing_file_price": "", + "user_group_list": null, + "is_restricted": false, + "is_robot": false, + "country": null, + "visitor_id": "e80a50a34252bc7ac5d6ea1e2fbc34bbc3eb3d98f92e31749ee06ef1", + "unique_session_id": "db731d4ba06907cb6ddcc9950e636eee5b2205e71e98af9004e9db2f", + "unique_id": "be175de4-34bd-3042-982a-1344ad93a8f0", + "hostname": "None" +} \ No newline at end of file diff --git a/modules/invenio-stats/tests/data/test_events/event_download03.json b/modules/invenio-stats/tests/data/test_events/event_download03.json new file mode 100644 index 0000000000..9e990c6a3d --- /dev/null +++ b/modules/invenio-stats/tests/data/test_events/event_download03.json @@ -0,0 +1,28 @@ +{ + "timestamp": "2024-01-01T00:00:00", + "bucket_id": "6fd627e8-e7ba-4837-8f75-54286575efda", + "file_id": "root_file_id_02", + "root_file_id": "root_file_id_02", + "file_key": "test_file_02.txt", + "size": 100, + "referrer": null, + "accessrole": "open_access", + "userrole": "System Administrator", + "site_license_name": "", + "site_license_flag": false, + "index_list": "利用報告", + "cur_user_id": 1, + "item_id": "1", + "item_title": "test", + "remote_addr": "192.168.56.1", + "is_billing_item": false, + "billing_file_price": "", + "user_group_list": null, + "is_restricted": false, + "is_robot": false, + "country": null, + "visitor_id": "e80a50a34252bc7ac5d6ea1e2fbc34bbc3eb3d98f92e31749ee06ef1", + "unique_session_id": "db731d4ba06907cb6ddcc9950e636eee5b2205e71e98af9004e9db2f", + "unique_id": "be175de4-34bd-3042-982a-1344ad93a8f0", + "hostname": "None" +} \ No newline at end of file diff --git a/modules/invenio-stats/tests/data/test_events/event_download04.json b/modules/invenio-stats/tests/data/test_events/event_download04.json new file mode 100644 index 0000000000..e321624d24 --- /dev/null +++ b/modules/invenio-stats/tests/data/test_events/event_download04.json @@ -0,0 +1,28 @@ +{ + "timestamp": "2024-01-31T23:59:59", + "bucket_id": "6fd627e8-e7ba-4837-8f75-54286575efda", + "file_id": "root_file_id_02", + "root_file_id": "root_file_id_02", + "file_key": "test_file_02.txt", + "size": 100, + "referrer": null, + "accessrole": "open_access", + "userrole": "System Administrator", + "site_license_name": "", + "site_license_flag": false, + "index_list": "利用報告", + "cur_user_id": 1, + "item_id": "1", + "item_title": "test", + "remote_addr": "192.168.56.1", + "is_billing_item": false, + "billing_file_price": "", + "user_group_list": null, + "is_restricted": false, + "is_robot": false, + "country": null, + "visitor_id": "e80a50a34252bc7ac5d6ea1e2fbc34bbc3eb3d98f92e31749ee06ef1", + "unique_session_id": "db731d4ba06907cb6ddcc9950e636eee5b2205e71e98af9004e9db2f", + "unique_id": "be175de4-34bd-3042-982a-1344ad93a8f0", + "hostname": "None" +} \ No newline at end of file diff --git a/modules/invenio-stats/tests/data/test_events/event_download05.json b/modules/invenio-stats/tests/data/test_events/event_download05.json new file mode 100644 index 0000000000..678b8fde2f --- /dev/null +++ b/modules/invenio-stats/tests/data/test_events/event_download05.json @@ -0,0 +1,28 @@ +{ + "timestamp": "2024-02-01T00:00:00", + "bucket_id": "6fd627e8-e7ba-4837-8f75-54286575efda", + "file_id": "root_file_id_02", + "root_file_id": "root_file_id_02", + "file_key": "test_file_02.txt", + "size": 100, + "referrer": null, + "accessrole": "open_access", + "userrole": "System Administrator", + "site_license_name": "", + "site_license_flag": false, + "index_list": "利用報告", + "cur_user_id": 1, + "item_id": "1", + "item_title": "test", + "remote_addr": "192.168.56.1", + "is_billing_item": false, + "billing_file_price": "", + "user_group_list": null, + "is_restricted": false, + "is_robot": false, + "country": null, + "visitor_id": "e80a50a34252bc7ac5d6ea1e2fbc34bbc3eb3d98f92e31749ee06ef1", + "unique_session_id": "db731d4ba06907cb6ddcc9950e636eee5b2205e71e98af9004e9db2f", + "unique_id": "be175de4-34bd-3042-982a-1344ad93a8f0", + "hostname": "None" +} \ No newline at end of file diff --git a/modules/invenio-stats/tests/test_queries.py b/modules/invenio-stats/tests/test_queries.py index 623f4f89f9..244400b0e2 100644 --- a/modules/invenio-stats/tests/test_queries.py +++ b/modules/invenio-stats/tests/test_queries.py @@ -324,3 +324,48 @@ def test_weko_terms_query(app): **test_config ) assert query.build_query(None, datetime.datetime(2023, 1, 1), after_key='test_key', required1='v1', agg_filter={'agg': 'agg1'}).to_dict() == {'query': {'bool': {'filter': [{'range': {'timestamp': {'lte': '2023-01-01T00:00:00', 'time_zone': 'Asia/Tokyo'}}}, {'term': {'required1': 'v1'}}, {'terms': {'agg': 'agg1'}}]}}, 'aggs': {'value': {'sum': {'field': 'count'}}, 'my_buckets': {'composite': {'size': 6000, 'sources': [{'search_key': {'terms': {'field': 'search_key'}}}, {'count': {'terms': {'field': 'count'}}}], 'after': 'test_key'}}}, 'from': 0, 'size': 0} + + +# .tox/c1/bin/pytest --cov=invenio_stats tests/test_queries.py::test_ESWekoFileRankingQuery -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-stats/.tox/c1/tmp +def test_ESWekoFileRankingQuery(app, esindex): + import json + from invenio_stats.proxies import current_stats + query_download_total_cfg = current_stats.queries['item-file-download-aggs'] + query_download_total = query_download_total_cfg.query_class(**query_download_total_cfg.query_config) + + index = app.config["INDEXER_DEFAULT_INDEX"] + doc_type = "stats-file-download" + + def register(i): + with open(f"tests/data/test_events/event_download{i:02}.json","r") as f: + esindex.index(index=index, doc_type=doc_type, id=f"{i}", body=json.load(f), refresh="true") + + def delete(i): + esindex.delete(index=index, doc_type=doc_type, id=f"{i}", refresh="true") + + # 19 Exist result + item_id = 1 + register(item_id) + params = { + 'item_id': str(item_id), + 'root_file_id_list': ['root_file_id_01'], + } + assert query_download_total.run(**params) == {'start_date': None, 'end_date': None, 'download_ranking': {'doc_count_error_upper_bound': 0, 'sum_other_doc_count': 0, 'buckets': [{'key': 'test_file_01.txt', 'doc_count': 1}]}} + delete(item_id) + + # 20 Not exist result + assert query_download_total.run(**params) == {'start_date': None, 'end_date': None, 'download_ranking': {'doc_count_error_upper_bound': 0, 'sum_other_doc_count': 0, 'buckets': []}} + + # 21 Set period + for i in range(2,6): + register(i) + params = { + 'item_id': str(item_id), + 'root_file_id_list': ['root_file_id_02'], + 'start_date': '2024-01-01', + 'end_date': '2024-01-31T23:59:59', + } + res = query_download_total.run(**params) + assert res['download_ranking']['buckets'][0]['doc_count'] == 2 + assert res['start_date'] == '2024-01-01T00:00:00' + assert res['end_date'] == '2024-01-31T23:59:59' diff --git a/modules/invenio-stats/tox.result.gz b/modules/invenio-stats/tox.result.gz index a06bee856e..284812816b 100644 Binary files a/modules/invenio-stats/tox.result.gz and b/modules/invenio-stats/tox.result.gz differ diff --git a/modules/weko-accounts/coverage.xml b/modules/weko-accounts/coverage.xml index f0e1b71c95..87c26d3458 100644 --- a/modules/weko-accounts/coverage.xml +++ b/modules/weko-accounts/coverage.xml @@ -1,12 +1,12 @@ - + /code/modules/weko-accounts/weko_accounts - + @@ -225,45 +225,83 @@ + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -296,25 +334,100 @@ - + + + + + + - - - - - + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -324,82 +437,87 @@ - - - - - + + + + + - - - - + + + + - - + + - - - + + + + - + - - + + - - - - + + + + - - + + - - - - + + + - + - - - - - + + + + + - + + - - - - - + + + + - + + + + + + diff --git a/modules/weko-accounts/requirements.txt b/modules/weko-accounts/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/weko-accounts/requirements.txt +++ b/modules/weko-accounts/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/weko-accounts/requirements2.txt b/modules/weko-accounts/requirements2.txt index f1ec27eb49..608fab3168 100644 --- a/modules/weko-accounts/requirements2.txt +++ b/modules/weko-accounts/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -193,9 +193,10 @@ PyLD==2.0.3 pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 -pytest>=4.2.0 -pytest-mock==3.6.1 -python-dateutil==2.6.1 +pytest==4.2.0 +pytest-mock +pytest-invenio +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +204,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/weko-accounts/setup.py b/modules/weko-accounts/setup.py index 721f304b88..b3eccbe33d 100644 --- a/modules/weko-accounts/setup.py +++ b/modules/weko-accounts/setup.py @@ -90,6 +90,9 @@ 'invenio_base.apps': [ 'weko_accounts = weko_accounts:WekoAccounts', ], + 'invenio_base.api_apps': [ + 'weko_accounts_rest = weko_accounts:WekoAccountsREST', + ], 'invenio_admin.views': [ 'weko_accounts_shibboleth = weko_accounts.admin:shib_adminview', ], diff --git a/modules/weko-accounts/tests/conftest.py b/modules/weko-accounts/tests/conftest.py index 6c92e2950c..5169f7a154 100644 --- a/modules/weko-accounts/tests/conftest.py +++ b/modules/weko-accounts/tests/conftest.py @@ -48,7 +48,7 @@ from weko_redis.redis import RedisConnection from weko_user_profiles import WekoUserProfiles -from weko_accounts import WekoAccounts +from weko_accounts import WekoAccounts, WekoAccountsREST from weko_accounts.views import blueprint @pytest.yield_fixture() @@ -94,6 +94,7 @@ def base_app(instance_path): WekoAdmin(app_) WekoUserProfiles(app_) app_.register_blueprint(blueprint) + WekoAccountsREST(app_) return app_ @@ -274,3 +275,10 @@ def session_time(app,db): def redis_connect(app): redis_connection = RedisConnection().connection(db=app.config['CACHE_REDIS_DB'], kv = True) return redis_connection + + +@pytest.fixture() +def users_login(users): + inactive_user = create_test_user(email='inactive_user@test.org', active=False) + users.append({'email': inactive_user.email, 'id': inactive_user.id, 'obj': inactive_user}) + return users diff --git a/modules/weko-accounts/tests/test_rest.py b/modules/weko-accounts/tests/test_rest.py new file mode 100644 index 0000000000..124d40b7a5 --- /dev/null +++ b/modules/weko-accounts/tests/test_rest.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +# +# This file is part of WEKO3. +# Copyright (C) 2017 National Institute of Informatics. +# +# WEKO3 is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# WEKO3 is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with WEKO3; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. + +"""Module tests.""" + +from flask import json + +from weko_accounts.errors import VersionNotFoundRESTError, UserAllreadyLoggedInError, UserNotFoundError, InvalidPasswordError, DisabledUserError + + +# .tox/c1/bin/pytest --cov=weko_accounts tests/test_rest.py::test_WekoLogin_post -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp +def test_WekoLogin_post(app, client, users_login): + """Test WekoLogin.post method.""" + + version = 'v1' + invalid_version = 'v0' + + # Invalid version : 400 error + req_json = { + 'email': users_login[5]['email'], + 'password': users_login[5]['obj'].password_plaintext, + } + res = client.post( + f'/{invalid_version}/login', + data=json.dumps(req_json), + content_type='application/json', + ) + res_data = json.loads(res.get_data()) + assert res.status_code == VersionNotFoundRESTError.code + assert res_data['message'] == VersionNotFoundRESTError.description + + # Invalid user : 403 error + req_json = { + 'email': 'dummy', + 'password': users_login[5]['obj'].password_plaintext, + } + res = client.post( + f'/{version}/login', + data=json.dumps(req_json), + content_type='application/json', + ) + res_data = json.loads(res.get_data()) + assert res.status_code == UserNotFoundError.code + assert res_data['message'] == UserNotFoundError.description + + # Invalid password : 403 error + req_json = { + 'email': users_login[5]['email'], + 'password': 'dummy', + } + res = client.post( + f'/{version}/login', + data=json.dumps(req_json), + content_type='application/json', + ) + res_data = json.loads(res.get_data()) + assert res.status_code == InvalidPasswordError.code + assert res_data['message'] == InvalidPasswordError.description + + # Inactive user : 403 error + req_json = { + 'email': users_login[9]['email'], + 'password': users_login[9]['obj'].password_plaintext, + } + res = client.post( + f'/{version}/login', + data=json.dumps(req_json), + content_type='application/json', + ) + res_data = json.loads(res.get_data()) + assert res.status_code == DisabledUserError.code + assert res_data['message'] == DisabledUserError.description + + # Normal : 200 + req_json = { + 'email': users_login[5]['email'], + 'password': users_login[5]['obj'].password_plaintext, + } + res = client.post( + f'/{version}/login', + data=json.dumps(req_json), + content_type='application/json', + ) + res_data = json.loads(res.get_data()) + assert res.status_code == 200 + assert res_data['id'] == users_login[5]['id'] + assert res_data['email'] == users_login[5]['email'] + + # User is already logged in : 400 error + req_json = { + 'email': users_login[5]['email'], + 'password': users_login[5]['obj'].password_plaintext, + } + res = client.post( + f'/{version}/login', + data=json.dumps(req_json), + content_type='application/json', + ) + res_data = json.loads(res.get_data()) + assert res.status_code == UserAllreadyLoggedInError.code + assert res_data['message'] == UserAllreadyLoggedInError.description + + +# .tox/c1/bin/pytest --cov=weko_accounts tests/test_rest.py::test_WekoLogout_post -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-accounts/.tox/c1/tmp +def test_WekoLogout_post(app, client, users_login): + """Test WekoLogout.post method.""" + + version = 'v1' + invalid_version = 'v0' + + # Invalid version : 400 error + res = client.post( + f'/{invalid_version}/logout', + data=json.dumps(None), + content_type='application/json', + ) + res_data = json.loads(res.get_data()) + assert res.status_code == VersionNotFoundRESTError.code + assert res_data['message'] == VersionNotFoundRESTError.description + + # Normal : 200 + req_json = { + 'email': users_login[5]['email'], + 'password': users_login[5]['obj'].password_plaintext, + } + client.post( + f'/{version}/login', + data=json.dumps(req_json), + content_type='application/json', + ) + res = client.post( + f'/{version}/logout', + data=json.dumps(None), + content_type='application/json', + ) + assert res.status_code == 200 + + # User is already logged out : 200 + res = client.post( + f'/{version}/logout', + data=json.dumps(None), + content_type='application/json', + ) + assert res.status_code == 200 diff --git a/modules/weko-accounts/tox.result.gz b/modules/weko-accounts/tox.result.gz index 03dbc45e68..9c4e06af58 100644 Binary files a/modules/weko-accounts/tox.result.gz and b/modules/weko-accounts/tox.result.gz differ diff --git a/modules/weko-accounts/weko_accounts/__init__.py b/modules/weko-accounts/weko_accounts/__init__.py index f313644c54..e8e0a07500 100644 --- a/modules/weko-accounts/weko_accounts/__init__.py +++ b/modules/weko-accounts/weko_accounts/__init__.py @@ -20,7 +20,7 @@ """Package initialization for weko-accounts.""" -from .ext import WekoAccounts +from .ext import WekoAccounts, WekoAccountsREST from .version import __version__ -__all__ = ('__version__', 'WekoAccounts') +__all__ = ('__version__', 'WekoAccounts', 'WekoAccountsREST') diff --git a/modules/weko-accounts/weko_accounts/config.py b/modules/weko-accounts/weko_accounts/config.py index 0029a17ec2..85c60a58b5 100644 --- a/modules/weko-accounts/weko_accounts/config.py +++ b/modules/weko-accounts/weko_accounts/config.py @@ -108,3 +108,18 @@ # WEKO_ACCOUNTS_REAL_IP = "x_forwarded_for" # X-Forwarded-For[first] > remote_addr # WEKO_ACCOUNTS_REAL_IP = "x_forwarded_for_rev" # X-Forwarded-For[last] > remote_addr +WEKO_ACCOUNTS_REST_ENDPOINTS = { + 'login': { + 'route': '//login', + 'default_media_type': 'application/json', + }, + 'logout': { + 'route': '//logout', + 'default_media_type': 'application/json', + }, +} + +WEKO_ACCOUNTS_API_LIMIT_RATE_DEFAULT = ['100 per minute'] + +WEKO_API_LIMIT_RATE_DEFAULT = ["100 per minute"] +"""Default rate limit per endpoint for one user in the WEKO API.""" diff --git a/modules/weko-accounts/weko_accounts/errors.py b/modules/weko-accounts/weko_accounts/errors.py new file mode 100644 index 0000000000..cdd4faee75 --- /dev/null +++ b/modules/weko-accounts/weko_accounts/errors.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +"""Records Error""" + +from invenio_rest.errors import RESTException + + +class VersionNotFoundRESTError(RESTException): + """API Version error.""" + + code = 400 + description = 'This API version does not found' + + +class UserAllreadyLoggedInError(RESTException): + """User allready logged in.""" + + code = 400 + description = 'User allready logged in.' + + +class UserNotFoundError(RESTException): + """User not found.""" + + code = 403 + description = 'Specified user does not exist.' + + +class InvalidPasswordError(RESTException): + """Invalid password.""" + + code = 403 + description = 'Invalid password.' + + +class DisabledUserError(RESTException): + """Account is disabled.""" + + code = 403 + description = 'Account is disabled.' diff --git a/modules/weko-accounts/weko_accounts/ext.py b/modules/weko-accounts/weko_accounts/ext.py index 50da78b041..088ae11ae6 100644 --- a/modules/weko-accounts/weko_accounts/ext.py +++ b/modules/weko-accounts/weko_accounts/ext.py @@ -24,8 +24,6 @@ from flask_login import user_logged_in, user_logged_out from . import config -from .sessions import login_listener, logout_listener -from .views import blueprint class WekoAccounts(object): @@ -53,9 +51,12 @@ def init_app(self, app): if app.config['WEKO_ACCOUNTS_LOGGER_ENABLED']: self._enable_logger_activity(app=app) + from .views import blueprint app.register_blueprint(blueprint) app.extensions['weko-accounts'] = self + self.init_limiter(app) + def init_config(self, app): """ Initialize configuration. @@ -103,5 +104,49 @@ def _enable_logger_activity(self, app): :param app: The flask application. """ + from .sessions import login_listener, logout_listener user_logged_in.connect(login_listener, app) user_logged_out.connect(logout_listener, app) + + def init_limiter(self, app): + """ + Initialize rate limiting. + + :param app: The flask application. + """ + from .utils import limiter + limiter.init_app(app) + + +class WekoAccountsREST(object): + """Weko accounts Rest Obj.""" + + def __init__(self, app=None): + """ + Extension initialization. + :param app: An instance of :class:`flask.Flask`. + """ + if app: + self.init_app(app) + + def init_app(self, app): + """ + Flask application initialization. + Initialize the REST endpoints. + Connect all signals if `DEPOSIT_REGISTER_SIGNALS` is True. + :param app: An instance of :class:`flask.Flask`. + """ + from .rest import create_blueprint + self.init_config(app) + blueprint = create_blueprint(app, app.config['WEKO_ACCOUNTS_REST_ENDPOINTS']) + app.register_blueprint(blueprint) + app.extensions['weko_accounts_rest'] = self + + def init_config(self, app): + """ + Initialize configuration. + :param app: An instance of :class:`flask.Flask`. + """ + for k in dir(config): + if k.startswith('WEKO_ACCOUNTS_'): + app.config.setdefault(k, getattr(config, k)) diff --git a/modules/weko-accounts/weko_accounts/rest.py b/modules/weko-accounts/weko_accounts/rest.py new file mode 100644 index 0000000000..a76bc97b78 --- /dev/null +++ b/modules/weko-accounts/weko_accounts/rest.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# +# This file is part of WEKO3. +# Copyright (C) 2017 National Institute of Informatics. +# +# WEKO3 is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# WEKO3 is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with WEKO3; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. + +"""Blueprint for Weko accounts rest.""" + +import inspect + +from flask import Blueprint, current_app, jsonify, request, make_response +from flask_login import login_user, logout_user +from flask_security import current_user +from flask_security.utils import verify_password + +from invenio_accounts.models import User +from invenio_db import db +from invenio_rest import ContentNegotiatedMethodView + +from .errors import VersionNotFoundRESTError, UserAllreadyLoggedInError, UserNotFoundError, InvalidPasswordError, DisabledUserError +from .utils import create_limiter + +limiter = create_limiter() + + +def create_blueprint(app, endpoints): + """ + Create Weko-Accounts-REST blueprint. + + See: :data:`weko_accounts.config.WEKO_ACCOUNTS_REST_ENDPOINTS`. + + :param endpoints: List of endpoints configuration. + :returns: The configured blueprint. + """ + blueprint = Blueprint( + 'weko_accounts_rest', + __name__, + url_prefix="", + ) + + @blueprint.teardown_request + def dbsession_clean(exception): + current_app.logger.debug('weko_accounts dbsession_clean: {}'.format(exception)) + if exception is None: + try: + db.session.commit() + except: + db.session.rollback() + db.session.remove() + + for endpoint, options in (endpoints or {}).items(): + if endpoint == 'login': + view_func = WekoLogin.as_view( + WekoLogin.view_name.format(endpoint), + default_media_type=options.get('default_media_type') + ) + blueprint.add_url_rule( + options.get('route'), + view_func=view_func, + methods=['POST'] + ) + elif endpoint == 'logout': + view_func = WekoLogout.as_view( + WekoLogout.view_name.format(endpoint), + default_media_type=options.get('default_media_type') + ) + blueprint.add_url_rule( + options.get('route'), + view_func=view_func, + methods=['POST'] + ) + + return blueprint + + +class WekoLogin(ContentNegotiatedMethodView): + """Resource to login as weko user.""" + + view_name = '{0}_accounts' + + def __init__(self, *args, **kwargs): + """Constructor.""" + super(WekoLogin, self).__init__(*args, **kwargs) + + @limiter.limit('') + def post(self, **kwargs): + """ + Login as weko user. + + Returns: + Login result. + """ + + version = kwargs.get('version') + func_name = f'post_{version}' + if func_name in [func[0] for func in inspect.getmembers(self, inspect.ismethod)]: + return getattr(self, func_name)(**kwargs) + else: + raise VersionNotFoundRESTError() + + def post_v1(self, **kwargs): + + data = request.get_json() + email = data['email'] + password = data['password'] + + # Check if user is already logged in + if current_user.is_authenticated: + raise UserAllreadyLoggedInError() + + # Get User + user = User.query.filter_by(email=email).first() + if not user: + raise UserNotFoundError() + # Verify password + if not verify_password(password, user.password): + raise InvalidPasswordError() + + # Check if user is active + if not user.active: + raise DisabledUserError() + + # Log in + login_user(user) + + # Create response + res_json = { + 'id': user.id, + 'email': user.email, + } + return make_response(jsonify(res_json), 200) + + +class WekoLogout(ContentNegotiatedMethodView): + """Resource to logout.""" + + view_name = '{0}_accounts' + + def __init__(self, *args, **kwargs): + """Constructor.""" + super(WekoLogout, self).__init__(*args, **kwargs) + + @limiter.limit('') + def post(self, **kwargs): + """ + Logout of weko. + + Returns: + Logout result. + """ + + version = kwargs.get('version') + func_name = f'post_{version}' + if func_name in [func[0] for func in inspect.getmembers(self, inspect.ismethod)]: + return getattr(self, func_name)(**kwargs) + else: + raise VersionNotFoundRESTError() + + def post_v1(self, **kwargs): + + # Logout + if current_user.is_authenticated: + logout_user() + + return make_response('', 200) diff --git a/modules/weko-accounts/weko_accounts/utils.py b/modules/weko-accounts/weko_accounts/utils.py index 0c31d7e87b..8660aae2ea 100644 --- a/modules/weko-accounts/weko_accounts/utils.py +++ b/modules/weko-accounts/weko_accounts/utils.py @@ -23,10 +23,20 @@ import string from functools import wraps -from flask import abort, current_app, request, session +from flask import abort, current_app, request, session, Flask +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address from flask_login import current_user from flask_login.config import EXEMPT_METHODS +from .config import WEKO_API_LIMIT_RATE_DEFAULT + +limiter = Limiter( + app=None, + key_func=lambda: f"{request.endpoint}_{get_remote_addr()}", + default_limits=WEKO_API_LIMIT_RATE_DEFAULT +) + def get_remote_addr(): """ @@ -41,7 +51,7 @@ def get_remote_addr(): # current_app.logger.debug('{0} {1} {2}: {3}'.format( # __file__, 'get_remote_addr()', 'request.headers', request.headers)) - + address = None if "WEKO_ACCOUNTS_REAL_IP" not in current_app.config or current_app.config["WEKO_ACCOUNTS_REAL_IP"] == None: address = request.headers.get('X-Real-IP', None) @@ -57,7 +67,7 @@ def get_remote_addr(): address = request.headers.get('X-Forwarded-For', None) if address is not None: _tmp = address.encode('utf-8').split(b',') - address = _tmp[0].strip().decode() + address = _tmp[0].strip().decode() elif current_app.config["WEKO_ACCOUNTS_REAL_IP"] == "x_forwarded_for_rev": address = request.headers.get('X-Forwarded-For', None) if address is not None: @@ -178,3 +188,8 @@ def decorated_view(*args, **kwargs): return func(*args, **kwargs) return decorated_view return decorator + + +def create_limiter(): + from .config import WEKO_ACCOUNTS_API_LIMIT_RATE_DEFAULT + return Limiter(app=Flask(__name__), key_func=get_remote_address, default_limits=WEKO_ACCOUNTS_API_LIMIT_RATE_DEFAULT) diff --git a/modules/weko-admin/requirements.txt b/modules/weko-admin/requirements.txt index 33d7a6bf5d..2e20530da5 100644 --- a/modules/weko-admin/requirements.txt +++ b/modules/weko-admin/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/weko-admin/requirements2.txt b/modules/weko-admin/requirements2.txt index 64f2dc7107..2b7292b7b6 100644 --- a/modules/weko-admin/requirements2.txt +++ b/modules/weko-admin/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -193,10 +193,10 @@ PyLD==2.0.3 pyparsing==3.1.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fefc684a3a74aff6f99e5dff24f9b4dd1c95169d#egg=PyPDF2 pyPEG2==2.15.2 -pytest==4.2.0 +pytest>=4.2.0 pytest-mock pytest-invenio -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -204,10 +204,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/weko-admin/setup.py b/modules/weko-admin/setup.py index 1abf39e8b7..947567e56c 100644 --- a/modules/weko-admin/setup.py +++ b/modules/weko-admin/setup.py @@ -132,6 +132,7 @@ 'weko_admin_item_export_settings = weko_admin.admin:item_export_settings_adminview', 'weko_admin_site_info = weko_admin.admin:site_info_settings_adminview', 'weko_admin_identifier = weko_admin.admin:identifier_adminview', + 'weko_admin_sword_api = weko_admin.admin:sword_api_settings_adminview', 'restricted_access_adminview = weko_admin.admin:restricted_access_adminview', 'facet_search_adminview = weko_admin.admin:facet_search_adminview', 'reindex_elasticsearch_adminview = weko_admin.admin:reindex_elasticsearch_adminview' diff --git a/modules/weko-admin/tests/conftest.py b/modules/weko-admin/tests/conftest.py index 552de57d38..a31ccd9d9b 100644 --- a/modules/weko-admin/tests/conftest.py +++ b/modules/weko-admin/tests/conftest.py @@ -755,6 +755,7 @@ def admin_settings(db): settings.append(AdminSettings(id=7,name="display_stats_settings",settings={"display_stats":False})) settings.append(AdminSettings(id=8,name='convert_pdf_settings',settings={"path":"/tmp/file","pdf_ttl":1800})) settings.append(AdminSettings(id=9,name="elastic_reindex_settings",settings={"has_errored": False})) + settings.append(AdminSettings(id=10,name="sword_api_setting",settings={ "default_format": "TSV","data_format":{ "TSV":{"register_format": "Direct"},"XML":{"workflow": '31001', "register_format": "Workflow"}}})) db.session.add_all(settings) db.session.commit() return settings diff --git a/modules/weko-admin/tests/test_admin.py b/modules/weko-admin/tests/test_admin.py index a7d2e4c777..d47b65903d 100644 --- a/modules/weko-admin/tests/test_admin.py +++ b/modules/weko-admin/tests/test_admin.py @@ -11,6 +11,9 @@ from datetime import datetime from wtforms.validators import ValidationError from werkzeug.datastructures import ImmutableMultiDict +from flask_wtf import FlaskForm,Form +from weko_workflow.api import WorkFlow +from weko_workflow.models import WorkFlow import pytest from requests import Response @@ -24,7 +27,7 @@ from weko_index_tree.models import IndexStyle,Index from weko_admin.admin import StyleSettingView,LogAnalysisSettings,ItemExportSettingsView,IdentifierSettingView,\ - identifier_adminview,facet_search_adminview,FacetSearchSettingView + identifier_adminview,facet_search_adminview,FacetSearchSettingView,SwordAPISettingsView from weko_admin.models import AdminSettings,StatisticsEmail,LogAnalysisRestrictedCrawlerList,\ RankingSettings,SearchManagement, Identifier,FacetSearchSetting @@ -1826,3 +1829,118 @@ def test_ReindexElasticSearchView_check_reindex_is_running_err(self, client,user res = client.get(url) assert res.status_code == 500 assert res.data != str(dict({ "isError":False ,"isExecuting":False,"disabled_Btn":False })) + +#class SwordAPISettingsView(BaseView): +# +class TestSwordAPISettingsView: +# def index(self): +# .tox/c1/bin/pytest --cov=weko_admin tests/test_admin.py::TestSwordAPISettingsView::test_index -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp --full-trace + + def test_index(self, client, users, db, admin_settings, mocker): + url = url_for("swordapi.index") + + #no administrator + res = client.get(url) + assert res.status_code == 302 + + login_user_via_session(client,email=users[7]["email"]) + res = client.get(url) + assert res.status_code == 403 + + #get + login_user_via_session(client,email=users[0]["email"])# sysadmin + deleted_workflow = WorkFlow( + id = "1", + flows_name = "test_workflow" + ) + mock_render = mocker.patch("weko_admin.admin.SwordAPISettingsView.render", return_value=make_response()) + with patch("weko_admin.admin.WorkFlow.get_workflows_by_roles", return_value=["workflow 1"]): + with patch("weko_admin.admin.WorkFlow.get_deleted_workflow_list", return_value=[deleted_workflow]): + res = client.get(url) + assert res.status_code == 200 + current_settings = { "default_format": "TSV", + "data_format":{ "TSV":{"register_format": "Direct"}, + "XML":{"workflow": '31001', "register_format": "Workflow"}}} + args, kwargs = mock_render.call_args + assert args[0] == "weko_admin/admin/sword_api_settings.html" + assert kwargs["current_settings"] == current_settings + assert json.loads(kwargs["current_settings_json"]) == current_settings + assert kwargs["deleted_workflow_name_dict"] == '{"1": "test_workflow"}' + assert kwargs["workflows"] == ["workflow 1"] + + + # not exist admin_settings + default_settings = { "default_format": "TSV", + "data_format":{ "TSV":{"register_format": "Direct"}, + "XML":{"workflow": "-1", "register_format": "Workflow"}}} + AdminSettings.query.filter_by(name="sword_api_setting").delete() + db.session.commit() + mock_render = mocker.patch("weko_admin.admin.SwordAPISettingsView.render", return_value = make_response()) + res = client.get(url) + assert res.status_code == 200 + args, kwargs = mock_render.call_args + assert args[0] == "weko_admin/admin/sword_api_settings.html" + assert kwargs["current_settings"] == default_settings + assert json.loads(kwargs["current_settings_json"]) == default_settings + assert kwargs["deleted_workflow_name_dict"] == '{}' + assert kwargs["workflows"] == [] + +# def default_format(self): +# .tox/c1/bin/pytest --cov=weko_admin tests/test_admin.py::TestSwordAPISettingsView::test_default_format -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp --full-trace + + def test_default_format(self,client,users,mocker,db,admin_settings): + login_user_via_session(client,email=users[0]["email"])# sysadmin + url = url_for('swordapi.default_format') + mocker.patch("weko_admin.admin.redirect",return_value=make_response()) + + data = {"default_format":"XML"} + res = client.post(url,data=json.dumps(data),headers={"content-type": "application/json"}) + assert res.status_code == 200 + setting = AdminSettings.get("sword_api_setting") + assert setting.default_format == "XML" + + with patch("weko_admin.admin.AdminSettings.update", side_effect = Exception("test_error")): + data = {"default_format":"TSV"} + res = client.post(url, data=json.dumps(data),headers={"content-type": "application/json"}) + assert res.status_code == 400 + setting = AdminSettings.get("sword_api_setting") + assert setting.default_format == "XML" + + +# def data_format(self): +# .tox/c1/bin/pytest --cov=weko_admin tests/test_admin.py::TestSwordAPISettingsView::test_data_format -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-admin/.tox/c1/tmp --full-trace + + def test_data_format(self,client, db, users, admin_settings, mocker): + url = url_for('swordapi.data_format') + login_user_via_session(client,email=users[0]["email"])# sysadmin + mocker.patch("weko_admin.admin.redirect",return_value=make_response()) + + data = { + "data_format": "TSV", + "register_format": "Direct", + "workflow": "" + } + res = client.post(url,json=data) + assert res.status_code == 200 + setting = AdminSettings.get("sword_api_setting") + assert setting.data_format["TSV"]["register_format"] == "Direct" + + data = { + "data_format": "XML", + "register_format": "Workflow", + "workflow": "-1" + } + res = client.post(url,json=data) + assert res.status_code == 200 + setting = AdminSettings.get("sword_api_setting") + assert setting.data_format["XML"]["register_format"] == "Workflow" + assert setting.data_format["XML"]["workflow"] == "-1" + + + with patch("weko_admin.admin.AdminSettings.update", side_effect=Exception("test_error")): + data = { "data_format": "XML", "register_format": "Workflow", "workflow": "31001"} + res = client.post(url, data=json.dumps(data),headers={"content-type": "application/json"}) + assert res.status_code == 400 + setting = AdminSettings.get("sword_api_setting") + assert setting.data_format["XML"]["register_format"] == "Workflow" + assert setting.data_format["XML"]["workflow"] == "-1" \ No newline at end of file diff --git a/modules/weko-admin/tox.result.gz b/modules/weko-admin/tox.result.gz index c612a8167e..d3953eb56c 100644 Binary files a/modules/weko-admin/tox.result.gz and b/modules/weko-admin/tox.result.gz differ diff --git a/modules/weko-admin/weko_admin/admin.py b/modules/weko-admin/weko_admin/admin.py index ea35ea2dc7..507284fdb0 100644 --- a/modules/weko-admin/weko_admin/admin.py +++ b/modules/weko-admin/weko_admin/admin.py @@ -50,6 +50,7 @@ from weko_records.models import SiteLicenseInfo from weko_records_ui.utils import check_items_settings from weko_schema_ui.models import PublishStatus +from weko_workflow.api import WorkFlow from wtforms.fields import StringField from wtforms.validators import ValidationError @@ -1367,7 +1368,74 @@ def delete(self, id=None): type_str='delete', id=id ) + +class SwordAPISettingsView(BaseView): + + @expose('/', methods=['GET']) + def index(self): + default_sword_api = { "default_format": "TSV", + "data_format":{ "TSV":{"register_format": "Direct"}, + "XML":{"workflow": "-1", "register_format": "Workflow"}}} # Default + current_settings = AdminSettings.get( + name='sword_api_setting', + dict_to_object=False) + if not current_settings: + AdminSettings.update('sword_api_setting', default_sword_api) + current_settings = AdminSettings.get( + name='sword_api_setting', + dict_to_object=False) + current_settings_json = json.dumps(current_settings) + form = FlaskForm(request.form) + workflow = WorkFlow() + workflow_list = workflow.get_workflow_list() + workflows = workflow.get_workflows_by_roles(workflow_list) + deleted_workflows = workflow.get_deleted_workflow_list() + deleted_workflow_name_dict = {} + for deleted_workflow in deleted_workflows: + deleted_workflow_name_dict[deleted_workflow.id] = deleted_workflow.flows_name + # Process exclude workflows + from weko_workflow.utils import exclude_admin_workflow + exclude_admin_workflow(workflows) + return self.render(current_app.config['WEKO_ADMIN_SWORD_API_TEMPLATE'], + current_settings = current_settings, + current_settings_json = current_settings_json, + deleted_workflow_name_dict = json.dumps(deleted_workflow_name_dict), + workflows = workflows, + form = form) + + @expose('/default_format', methods=['POST']) + def default_format(self): + """Default Format.""" + try: + settings = AdminSettings.get('sword_api_setting') + default_format_setting = request.json.get('default_format') + settings.default_format = default_format_setting + AdminSettings.update('sword_api_setting', + settings.__dict__) + return jsonify(success=True),200 + except Exception as e: + current_app.logger.error( + 'ERROR Default Form Settings: {}'.format(e)) + return jsonify(success=False),400 + @expose('/data_format', methods=['POST']) + def data_format(self): + """Data Format Settings.""" + try: + settings = AdminSettings.get('sword_api_setting') + data_format_setting = request.json.get('data_format') + if data_format_setting == "XML": + workflow_setting = request.json.get('workflow') + settings.data_format = {"TSV": {"register_format": "Direct"}, "XML": {"workflow": workflow_setting, "register_format": "Workflow"}} + else: + settings.data_format["TSV"]["register_format"] = "Direct" + AdminSettings.update('sword_api_setting', + settings.__dict__) + return jsonify(success=True),200 + except Exception as e: + current_app.logger.error( + 'ERROR Default Form Settings: {}'.format(e)) + return jsonify(success=False),400 style_adminview = { 'view_class': StyleSettingView, @@ -1528,6 +1596,14 @@ def delete(self, id=None): } } +sword_api_settings_adminview = { + 'view_class': SwordAPISettingsView, + 'kwargs': { + 'category': _('Setting'), + 'name': _('SWORD API'), + 'endpoint': 'swordapi' + } +} __all__ = ( 'style_adminview', @@ -1547,5 +1623,6 @@ def delete(self, id=None): 'restricted_access_adminview', 'identifier_adminview', 'facet_search_adminview', - 'reindex_elasticsearch_adminview' + 'reindex_elasticsearch_adminview', + 'sword_api_settings_adminview' ) diff --git a/modules/weko-admin/weko_admin/config.py b/modules/weko-admin/weko_admin/config.py index ef2dc8038f..dc494f58e4 100644 --- a/modules/weko-admin/weko_admin/config.py +++ b/modules/weko-admin/weko_admin/config.py @@ -51,6 +51,9 @@ WEKO_ADMIN_SITE_LICENSE_SEND_MAIL_TEMPLATE = 'weko_admin/admin/site_license_send_mail_settings.html' """Site-license send mail settings templates.""" +WEKO_ADMIN_SWORD_API_TEMPLATE = 'weko_admin/admin/sword_api_settings.html' +"""SWORD API template.""" + WEKO_ADMIN_BlOCK_STYLE_TEMPLATE = 'weko_admin/admin/block_style.html' """Block-style template.""" @@ -1144,6 +1147,7 @@ 'sitelicensesettings', 'itemtypesregister', 'itemtypesmapping', + 'itemtypesrocratemapping', 'itemtypes/mapping', 'items/bulk-export', 'feedbackmail', diff --git a/modules/weko-admin/weko_admin/static/css/weko_admin/styles.css b/modules/weko-admin/weko_admin/static/css/weko_admin/styles.css index e7fb983744..ec592ef708 100644 --- a/modules/weko-admin/weko_admin/static/css/weko_admin/styles.css +++ b/modules/weko-admin/weko_admin/static/css/weko_admin/styles.css @@ -118,3 +118,7 @@ label.field-required::after { .popup-mail-modal { width: 70%; } + +.errorMessage{ + color:red; +} \ No newline at end of file diff --git a/modules/weko-admin/weko_admin/static/js/weko_admin/facet_search_admin.js b/modules/weko-admin/weko_admin/static/js/weko_admin/facet_search_admin.js index b0629cd88e..e92ead8c79 100644 --- a/modules/weko-admin/weko_admin/static/js/weko_admin/facet_search_admin.js +++ b/modules/weko-admin/weko_admin/static/js/weko_admin/facet_search_admin.js @@ -498,7 +498,7 @@ function CustomAggregations({aggregations, setAggregations, mappingList}) { onChange={e => setAggMapping(e.target.value)}> { mappingList.map((item) => - ) + ) } diff --git a/modules/weko-admin/weko_admin/static/js/weko_admin/sword_api_settings.js b/modules/weko-admin/weko_admin/static/js/weko_admin/sword_api_settings.js new file mode 100644 index 0000000000..86820a7d56 --- /dev/null +++ b/modules/weko-admin/weko_admin/static/js/weko_admin/sword_api_settings.js @@ -0,0 +1,148 @@ +const Required_item = document.getElementById("Required_item").value; +const Successfully_Changed = document.getElementById("Successfully_Changed").value; +const Failed_Changed = document.getElementById("Failed_Changed").value; +const item_required_alert = document.getElementById("items_required_alert").value; +const workflow_deleted_alert = document.getElementById("workflow_deleted_alert").value; +let settings = document.getElementById("current_settings_json").value; +settings = JSON.parse(settings); +let deleted_workflows_name = document.getElementById("deleted_workflows_name").value; +deleted_workflows_name = JSON.parse(deleted_workflows_name); + +/** close ErrorMessage area */ +function closeError() { + $('#errors').empty(); +} + +/** show ErrorMessage */ +function showMsg(msg , success=false) { + $('#errors').append( + '
' + + '' + msg + '
'); +} + +function componentDidMount() { + /** set errorMessage Area */ + const header = document.getElementsByClassName('content-header')[0]; + if (header) { + const errorElement = document.createElement('div'); + errorElement.setAttribute('id', 'errors'); + header.insertBefore(errorElement, header.firstChild); + } +} + +function isDeletedWorkflow(value){ + var is_deleted = false; + var keys = Object.keys(deleted_workflows_name) + if(keys.includes(value)){ + is_deleted = true; + } + return is_deleted; +} + +function toggleMenu() { + const dataMenu = document.getElementById("data_format"); + const registerMenu = document.getElementById("register_format"); + const workflowMenu = document.getElementById("workflow"); + const workflowOption = document.createElement('option'); + closeError(); + + if (dataMenu.value === "empty") { + // 最初のメニューが空欄の場合、二番目のメニューを非活性化し、空欄を選択状態にする + // 最初のメニューが選択された場合、二番目のメニューを活性化する + registerMenu.disabled = true; + registerMenu.value = "empty"; + workflowMenu.disabled = true; + workflowMenu.value = "empty"; + } else if(dataMenu.value === "TSV"){ + registerMenu.value ="Direct"; + workflowMenu.value = ""; + workflowMenu.disabled = true; + } else if(dataMenu.value === "XML"){ + registerMenu.value = "Workflow"; + workflowMenu.removeAttribute("disabled"); + workflowMenu.setAttribute("required",true); + if(isDeletedWorkflow(settings["data_format"]["XML"]["workflow"])){ + workflowOption.value = "deleted_workflow"; + workflowOption.textContent = deleted_workflows_name[settings["data_format"]["XML"]["workflow"]] + "(削除済みワークフロー)"; + workflowOption.selected = true; + workflowOption.setAttribute("hidden","hidden"); + workflowMenu.appendChild(workflowOption); + return showMsg(workflow_deleted_alert, false); + }else{ + workflowMenu.value = settings["data_format"]["XML"]["workflow"]; + } + }else{ + registerMenu.removeAttribute("disabled"); + workflowOption.removeAttribute("disabled"); + registerMenu.setAttribute("required",true); + workflowMenu.setAttribute("required",true); + } +} + + +function isEmpty(value){ + if (!value){ + return true; + }else{ + return false; + } +} + +function handleDefaultSubmit() { + const form ={ + 'default_format': document.getElementById("default_select").value + } + closeError(); + fetch("/admin/swordapi/default_format" ,{method:'POST' ,headers:{'Content-Type':'application/json'} ,credentials:"include", body: JSON.stringify(form)}) + .then(res => { + if(!res.ok){ + console.log(etext); + } + showMsg(Successfully_Changed , true); + }) + .catch(error => { + console.log(error); + showMsg(Failed_Changed , false); + }); +} + +function handleSubmit(event) { + const dataMenu = document.getElementById("data_format"); + const registerMenu = document.getElementById("register_format"); + const workflowMenu = document.getElementById("workflow"); + closeError(); + + //Validate + // required check + NGList = []; + if(isEmpty(dataMenu.value)){ + NGList.push('Data Format'); + } + if(NGList.length){ + return showMsg(item_required_alert + NGList , false); + } + if(workflowMenu.value === "deleted_workflow") { + return showMsg(workflow_deleted_alert, false); + } + + + const form = { + 'data_format': document.getElementById("data_format").value + ,'register_format':document.getElementById("register_format").value + ,'workflow':document.getElementById("workflow").value + } + + fetch("/admin/swordapi/data_format" ,{method:'POST' ,headers:{'Content-Type':'application/json'} ,credentials:"include", body: JSON.stringify(form)}) + .then(res => { + if(!res.ok){ + console.log(etext); + } + showMsg(Successfully_Changed , true); + }) + .catch(error => { + console.log(error); + showMsg(Failed_Changed , false); + }); +} +componentDidMount(); \ No newline at end of file diff --git a/modules/weko-admin/weko_admin/templates/weko_admin/admin/sword_api_settings.html b/modules/weko-admin/weko_admin/templates/weko_admin/admin/sword_api_settings.html new file mode 100644 index 0000000000..16cf3ce9f5 --- /dev/null +++ b/modules/weko-admin/weko_admin/templates/weko_admin/admin/sword_api_settings.html @@ -0,0 +1,143 @@ +{# -*- coding: utf-8 -*- +# +# This file is file is part of WEKO3. +# Copyright (C) 2017 National Institute of Informatics. +# +# WEKO3 is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# WEKO3 is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with WEKO3; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +#} + +{%- extends admin_base_template %} + +{% block css %} +{{ super() }} + + + +{% endblock %} + +{%- block page_body_tabs %} + {% from "weko_theme/macros/edit_tabs_selector.html" import tabs_selector with context %} + {{ tabs_selector('item_type') }} +{%- endblock page_body_tabs%} + +{%- block javascript %} + {{ super() }} + +{%- endblock javascript %} + +{%- block body %} +
+ + + + + + + + + +
+
+
+

Default Format

+

+
+
+
+ {{ form.csrf_token }} +
+
+ +
+
+ +
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+

Data Format

+

+
+
+
+ {{ form.csrf_token }} +
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+
+
+
+ +
+
+
+
+
+{%- endblock body %} diff --git a/modules/weko-admin/weko_admin/translations/en/LC_MESSAGES/messages.po b/modules/weko-admin/weko_admin/translations/en/LC_MESSAGES/messages.po index bfdb2b431b..eab1fe4025 100644 --- a/modules/weko-admin/weko_admin/translations/en/LC_MESSAGES/messages.po +++ b/modules/weko-admin/weko_admin/translations/en/LC_MESSAGES/messages.po @@ -1359,6 +1359,10 @@ msgstr "" msgid "Favicon" msgstr "" +#: weko_admin/admin.py:1359 +msgid "The selected Workflow has been deleted. Please recheck and input." +msgstr "" + #: weko_admin/templates/weko_admin/admin/site_info.html:41 msgid "CopyRight" msgstr "" diff --git a/modules/weko-admin/weko_admin/translations/ja/LC_MESSAGES/messages.po b/modules/weko-admin/weko_admin/translations/ja/LC_MESSAGES/messages.po index 6bcd3c184b..2ee428c278 100644 --- a/modules/weko-admin/weko_admin/translations/ja/LC_MESSAGES/messages.po +++ b/modules/weko-admin/weko_admin/translations/ja/LC_MESSAGES/messages.po @@ -286,6 +286,16 @@ msgstr "" msgid "Faceted Search" msgstr "ファセット検索一覧" +#: weko_admin/admin.py:1357 +#: weko_admin/templates/weko_admin/admin/sword_api_setting.html:111 +msgid "Required item is not inputted" +msgstr "必須項目がありません。" + +#: weko_admin/admin.py:1359 +#: weko_admin/templates/weko_admin/admin/sword_api_setting.html:113 +msgid "The selected Workflow has been deleted. Please recheck and input." +msgstr "選択されたワークフローは削除されました。ご確認の上、再度入力してください。" + #: weko_admin/api.py:82 #, fuzzy msgid "statistics report" diff --git a/modules/weko-admin/weko_admin/utils.py b/modules/weko-admin/weko_admin/utils.py index bf70faf19d..2179b1c52d 100644 --- a/modules/weko-admin/weko_admin/utils.py +++ b/modules/weko-admin/weko_admin/utils.py @@ -961,7 +961,7 @@ def build_mail_data_to_string(cls, data, system_default_language): result += '[閲覧回数] : ' + str( cls.convert_download_count_to_int( item['detail_view'])) + '\n' - result += '[ファイルダウンロード回数] : ' + file_down_str + result += '[ファイルダウンロード回数] : \n' + file_down_str else: result += '[Title] : ' + item['title'] + '\n' diff --git a/modules/weko-admin/weko_admin/views.py b/modules/weko-admin/weko_admin/views.py index 7dcd4a1bc7..6852c7a52b 100644 --- a/modules/weko-admin/weko_admin/views.py +++ b/modules/weko-admin/weko_admin/views.py @@ -39,7 +39,7 @@ from sqlalchemy.orm import session from weko_accounts.utils import roles_required from weko_records.models import SiteLicenseInfo -from weko_index_tree.utils import delete_index_trees_from_redis + from werkzeug.local import LocalProxy from .api import send_site_license_mail @@ -219,6 +219,7 @@ def save_lang_list(): return jsonify(msg='Header Error') result = 'success' data = request.get_json() + from weko_index_tree.utils import delete_index_trees_from_redis for lang_code in [lang["lang_code"] for lang in data if not lang["is_registered"]]: delete_index_trees_from_redis(lang_code) diff --git a/modules/weko-authors/requirements.txt b/modules/weko-authors/requirements.txt index 8178d7d593..8f44203c02 100644 --- a/modules/weko-authors/requirements.txt +++ b/modules/weko-authors/requirements.txt @@ -13,6 +13,7 @@ binaryornot==0.4.4 bleach==3.1.0 blinker==1.4 cachelib==0.1 +captcha==0.4 cchardet==2.1.1 celery>=4.4.4 certifi==2017.11.5 @@ -114,7 +115,7 @@ PyJWT==1.5.3 #PyPDF2==1.26.0 -e git+https://github.com/RCOSDP/PyPDF2.git@fix/nii#egg=PyPDF2 pyPEG2==2.15.2 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -122,6 +123,7 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 @@ -161,7 +163,7 @@ boto3==1.7.84 s3fs==0.1.6 #fpdf==1.7.2 -e git+https://github.com/RCOSDP/pyfpdf.git@fix/nii#egg=fpdf -Pillow==5.4.1 +Pillow==8.1.2 Sickle==0.6.4 Sphinx==1.8.4 pytest>=4.2.0 diff --git a/modules/weko-authors/requirements2.txt b/modules/weko-authors/requirements2.txt index f1ec27eb49..41c12fbcfe 100644 --- a/modules/weko-authors/requirements2.txt +++ b/modules/weko-authors/requirements2.txt @@ -19,6 +19,7 @@ blinker==1.4 boto3==1.7.84 botocore==1.10.84 cachelib==0.1 +captcha==0.4 cachetools==4.2.4 cchardet==2.1.1 celery==4.4.4 @@ -177,7 +178,6 @@ passlib==1.7.1 pbr==5.11.1 pexpect==4.3.0 pickleshare==0.7.4 -Pillow==5.4.1 platformdirs==2.4.0 pluggy==0.13.1 ply==3.11 @@ -195,7 +195,7 @@ pyparsing==3.1.0 pyPEG2==2.15.2 pytest>=4.2.0 pytest-mock==3.6.1 -python-dateutil==2.6.1 +python-dateutil==2.8.2 python-editor==1.0.3 python-geoip==1.2 pytz==2017.3 @@ -203,10 +203,12 @@ pyzmq==17.0.0 redis==2.10.6 requests==2.18.4 requests-oauthlib==1.1.0 +rocrate==0.6.0 resync==2.0.1 s3fs==0.1.6 s3transfer==0.1.13 selenium +Pillow==8.1.2 Sickle==0.6.4 simplegeneric==0.8.1 simplejson==3.12.0 diff --git a/modules/weko-authors/setup.py b/modules/weko-authors/setup.py index 0606e5e353..960cc55c01 100644 --- a/modules/weko-authors/setup.py +++ b/modules/weko-authors/setup.py @@ -85,6 +85,10 @@ 'invenio_base.apps': [ 'weko_authors = weko_authors:WekoAuthors', ], + 'invenio_base.api_apps': [ + 'weko_authors = weko_authors:WekoAuthors', + 'weko_authors_rest = weko_authors:WekoAuthorsREST', + ], 'invenio_admin.views': [ 'weko_authors_management = ' 'weko_authors.admin:authors_list_adminview', @@ -96,9 +100,6 @@ 'invenio_base.blueprints': [ 'weko_authors = weko_authors.views:blueprint', ], - 'invenio_base.api_apps': [ - 'weko_authors = weko_authors:WekoAuthors', - ], 'invenio_base.api_blueprints': [ 'weko_authors = weko_authors.views:blueprint_api', ], @@ -131,6 +132,12 @@ 'invenio_access.actions': [ 'author_access = weko_authors.permissions:action_author_access', ], + 'invenio_oauth2server.scopes': [ + 'oauth_author_search_scope = weko_authors.scopes:author_search_scope', + 'oauth_author_create_scope = weko_authors.scopes:author_create_scope', + 'oauth_author_update_scope = weko_authors.scopes:author_update_scope', + 'oauth_author_delete_scope = weko_authors.scopes:author_delete_scope', + ], }, extras_require=extras_require, install_requires=install_requires, diff --git a/modules/weko-authors/tests/conftest.py b/modules/weko-authors/tests/conftest.py index 8a73c9c2f0..c8f003e169 100644 --- a/modules/weko-authors/tests/conftest.py +++ b/modules/weko-authors/tests/conftest.py @@ -1,22 +1,10 @@ # -*- coding: utf-8 -*- # -# This file is part of WEKO3. +# This file is part of Invenio. # Copyright (C) 2017 National Institute of Informatics. # -# WEKO3 is free software; you can redistribute it -# and/or modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# WEKO3 is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with WEKO3; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, -# MA 02111-1307, USA. +# Invenio is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. """Pytest configuration.""" import os,sys @@ -25,7 +13,7 @@ import json import uuid from os.path import dirname, join - +from datetime import datetime from elasticsearch import Elasticsearch from sqlalchemy import inspect @@ -44,20 +32,28 @@ from invenio_assets import InvenioAssets from invenio_cache import InvenioCache from invenio_communities.models import Community -from invenio_db import InvenioDB, db as db_ +# from invenio_db import InvenioDB, db as db_ +from invenio_db import InvenioDB +from invenio_oauth2server import InvenioOAuth2Server, InvenioOAuth2ServerREST +from invenio_db import db as db_ from invenio_files_rest import InvenioFilesREST from invenio_files_rest.models import Location, FileInstance from invenio_indexer import InvenioIndexer from invenio_search import InvenioSearch,RecordsSearch +from weko_authors.config import WEKO_AUTHORS_REST_ENDPOINTS from weko_search_ui import WekoSearchUI from weko_index_tree.models import Index - +from flask_limiter import Limiter +from flask_menu import Menu from weko_authors.views import blueprint_api +from weko_authors.rest import create_blueprint from weko_authors import WekoAuthors from weko_authors.models import Authors, AuthorsPrefixSettings, AuthorsAffiliationSettings +from weko_accounts import WekoAccounts from weko_theme import WekoTheme import weko_authors.mappings.v2 + sys.path.append(os.path.dirname(__file__)) class TestSearch(RecordsSearch): @@ -155,8 +151,17 @@ def base_app(request, instance_path,search_class): WEKO_AUTHORS_AFFILIATION_IDENTIFIER_ITEM_OTHER=4, WEKO_AUTHORS_LIST_SCHEME_AFFILIATION=[ 'ISNI', 'GRID', 'Ringgold', 'kakenhi', 'Other'], + WEKO_API_LIMIT_RATE_DEFAULT=["100 per minute"], CELERY_ALWAYS_EAGER=True, CELERY_CACHE_BACKEND="memory", + WEKO_AUTHORS_IMPORT_CACHE_RESULT_OVER_MAX_FILE_PATH_KEY = "authors_import_result_file_of_over_path", + WEKO_AUTHORS_EXPORT_TEMP_FOLDER_PATH = "/var/tmp/authors_export", + WEKO_AUTHORS_IMPORT_CACHE_USER_TSV_FILE_KEY = 'authors_import_user_file_key', + WEKO_AUTHORS_IMPORT_TEMP_FOLDER_PATH = "var/tmp/authors_import", + WEKO_AUTHORS_FILE_MAPPING_FOR_PREFIX =["scheme", "name", "url", "is_deleted"], + WEKO_AUTHORS_IMPORT_TMP_PREFIX = 'authors_import_', + WEKO_AUTHORS_IMPORT_BATCH_SIZE = 100, + WEKO_AUTHORS_IMPORT_MAX_NUM_OF_DISPLAYS = 1000, CELERY_EAGER_PROPAGATES_EXCEPTIONS=True, CELERY_RESULT_BACKEND="cache", CACHE_REDIS_URL=os.environ.get("CACHE_REDIS_URL", "redis://redis:6379/0"), @@ -165,8 +170,16 @@ def base_app(request, instance_path,search_class): SEARCH_ELASTIC_HOSTS=os.environ.get("INVENIO_ELASTICSEARCH_HOST"), SEARCH_INDEX_PREFIX="{}-".format('test'), SEARCH_CLIENT_CONFIG=dict(timeout=120, max_retries=10), + WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY="weko_authors_export_target", + WEKO_AUTHORS_EXPORT_CACHE_STOP_POINT_KEY="weko_authors_export_stop_point", + WEKO_AUTHORS_EXPORT_CACHE_TEMP_FILE_PATH_KEY="weko_authors_export_temp_file_path_key", + WEKO_AUTHORS_IMPORT_CACHE_BAND_CHECK_USER_FILE_PATH_KEY = "authors_import_band_check_user_file_path", + WEKO_AUTHORS_IMPORT_CACHE_RESULT_FILE_PATH_KEY = "authors_import_result_file_path", + WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY= "result_summary_key", + WEKO_AUTHORS_IMPORT_CACHE_OVER_MAX_TASK_KEY = "authors_import_over_max_task", ) Babel(app_) + Menu(app_) InvenioDB(app_) InvenioCache(app_) InvenioAccounts(app_) @@ -184,6 +197,10 @@ def base_app(request, instance_path,search_class): WekoTheme(app_) WekoAuthors(app_) WekoSearchUI(app_) + WekoAccounts(app_) + InvenioOAuth2Server(app_) + InvenioOAuth2ServerREST(app_) + # app_.register_blueprint(blueprint) app_.register_blueprint(blueprint_api, url_prefix='/api/authors') @@ -197,15 +214,217 @@ def app(base_app): yield base_app +@pytest.fixture() +def base_app2(instance_path,search_class): + """Flask application fixture for ES.""" + app_ = Flask('testapp', instance_path=instance_path) + WEKO_AUTHORS_FILE_MAPPING_FOR_AFFILIATION ={ + "json_id": "affiliationInfo", + "child": [ + { + "json_id": "identifierInfo", + "child": [ + { + "json_id": "affiliationIdType", + "label_en": "Affiliation Identifier Scheme", + "label_jp": "外部所属機関ID 識別子", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_affiliation_identifier_scheme" + }, + "required": { + "if": [ + "affiliationId" + ] + } + } + }, + { + "json_id": "affiliationId", + "label_en": "Affiliation Identifier", + "label_jp": "外部所属機関ID", + "validation": { + "required": { + "if": [ + "affiliationIdType" + ] + } + } + }, + { + "json_id": "identifierShowFlg", + "label_en": "Affiliation Identifier Display", + "label_jp": "外部所属機関ID 表示／非表示", + "mask": { + "true": "Y", + "false": "N" + } + } + ] + }, + { + "json_id": "affiliationNameInfo", + "child": [ + { + "json_id": "affiliationName", + "label_en": "Affiliation Name", + "label_jp": "外部所属機関名" + }, + { + "json_id": "affiliationNameLang", + "label_en": "Language", + "label_jp": "言語", + "validation": { + "map": [ + "ja", + "ja-Kana", + "en", + "fr", + "it", + "de", + "es", + "zh-cn", + "zh-tw", + "ru", + "la", + "ms", + "eo", + "ar", + "el", + "ko" + ] + } + }, + { + "json_id": "affiliationNameShowFlg", + "label_en": "Affiliation Name Display", + "label_jp": "外部所属機関名・言語 表示／非表示", + "mask": { + "true": "Y", + "false": "N" + }, + "validation": { + "map": [ + "Y", + "N" + ] + } + } + ] + }, + { + "json_id": "affiliationPeriodInfo", + "child": [ + { + "json_id": "periodStart", + "label_en": "Affiliation Period Start", + "label_jp": "外部所属機関 所属期間 開始日", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_affiliation_period_start" + } + } + }, + { + "json_id": "periodEnd", + "label_en": "Affiliation Period End", + "label_jp": "外部所属機関 所属期間 終了日", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_affiliation_period_end" + } + } + } + ] + } + ] + } + + app_.config.update( + SECRET_KEY='SECRET_KEY', + TESTING=True, + SERVER_NAME='app2', + SQLALCHEMY_DATABASE_URI=os.environ.get( + 'SQLALCHEMY_DATABASE_URI', + 'postgresql+psycopg2://invenio:dbpass123@postgresql:5432/wekotest'), + # SQLALCHEMY_DATABASE_URI=os.environ.get( + # 'SQLALCHEMY_DATABASE_URI', + # 'postgresql+psycopg2://invenio:dbpass123@postgresql:5432/invenio'), + # SQLALCHEMY_DATABASE_URI=os.environ.get( + # 'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'), + SQLALCHEMY_TRACK_MODIFICATIONS=True, + INDEX_IMG='indextree/36466818-image.jpg', + INDEXER_DEFAULT_INDEX="{}-authors-author-v1.0.0".format( + 'test' + ), + SEARCH_UI_SEARCH_INDEX='test-weko', + WEKO_AUTHORS_ES_INDEX_NAME='test-authors', + WEKO_AUTHORS_AFFILIATION_IDENTIFIER_ITEM_OTHER=4, + WEKO_AUTHORS_LIST_SCHEME_AFFILIATION=[ + 'ISNI', 'GRID', 'Ringgold', 'kakenhi', 'Other'], + CELERY_ALWAYS_EAGER=True, + CELERY_CACHE_BACKEND="memory", + CELERY_EAGER_PROPAGATES_EXCEPTIONS=True, + CELERY_RESULT_BACKEND="cache", + CACHE_REDIS_URL='redis://redis:6379/0', + CACHE_REDIS_DB='0', + CACHE_REDIS_HOST="redis", + SEARCH_ELASTIC_HOSTS=os.environ.get("SEARCH_ELASTIC_HOSTS", "elasticsearch"), + WEKO_AUTHORS_EXPORT_BATCH_SIZE=2, + WEKO_AUTHORS_FILE_MAPPING_FOR_AFFILIATION=WEKO_AUTHORS_FILE_MAPPING_FOR_AFFILIATION, + WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY=2, + WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL=1, + WEKO_AUTHORS_IMPORT_TEMP_FOLDER_PATH='/data', + WEKO_AUTHORS_CACHE_TTL=100, + WEKO_AUTHORS_IMPORT_BATCH_SIZE=2, + WEKO_AUTHORS_IMPORT_MAX_RETRY=2, + WEKO_AUTHORS_IMPORT_RETRY_INTERVAL=1, + WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY= "result_summary_key", + WEKO_AUTHORS_BULK_IMPORT_RETRY_INTERVAL= 1, + WEKO_AUTHORS_EXPORT_CACHE_URL_KEY= 'weko_authors_exported_url', + WEKO_AUTHORS_EXPORT_CACHE_STOP_POINT_KEY= 'weko_authors_export_stop_point', + WEKO_AUTHORS_IMPORT_CACHE_FORCE_CHANGE_MODE_KEY= 'authors_import_force_change', + ) + Babel(app_) + InvenioDB(app_) + InvenioCache(app_) + InvenioAccounts(app_) + InvenioAccess(app_) + InvenioAdmin(app_) + InvenioAssets(app_) + InvenioIndexer(app_) + InvenioFilesREST(app_) + + search = InvenioSearch(app_) + search.register_mappings(search_class.Meta.index, 'mock_module.mapping') + WekoTheme(app_) + WekoAuthors(app_) + WekoSearchUI(app_) + + # app_.register_blueprint(blueprint) + app_.register_blueprint(blueprint_api, url_prefix='/api/authors') + return app_ + + +@pytest.yield_fixture() +def app2(base_app2): + """Flask application fixture.""" + with base_app2.app_context(): + yield base_app2 + + @pytest.yield_fixture() def db(app): """Database fixture.""" + db_.session.remove() + db_.drop_all() if not database_exists(str(db_.engine.url)): create_database(str(db_.engine.url)) db_.create_all() yield db_ - db_.session.remove() - db_.drop_all() # drop_database(str(db_.engine.url)) @@ -227,9 +446,9 @@ def esindex(app): yield current_search_client - with app.test_request_context(): - current_search_client.indices.delete_alias(index="test-authors-author-v1.0.0", name=app.config["WEKO_AUTHORS_ES_INDEX_NAME"]) - current_search_client.indices.delete(index="test-authors-author-v1.0.0", ignore=[400, 404]) + # with app.test_request_context(): + # current_search_client.indices.delete_alias(index="test-authors-author-v1.0.0", name=app.config["WEKO_AUTHORS_ES_INDEX_NAME"]) + # current_search_client.indices.delete(index="test-authors-author-v1.0.0", ignore=[400, 404]) @pytest.fixture() @@ -410,6 +629,30 @@ def json_data(filename): def authors(app,db,esindex): datas = json_data("data/author.json") returns = list() + for data in datas: + returns.append(Authors( + gather_flg=data.get("gather_flg", 0), + is_deleted=data.get("is_deleted", False), + json=data + )) + es_id = data["id"] + es_data = json.loads(json.dumps(data)) + es_data["id"]="" + current_search_client.index( + index=app.config["WEKO_AUTHORS_ES_INDEX_NAME"], + doc_type=app.config['WEKO_AUTHORS_ES_DOC_TYPE'], + id=es_id, + body=es_data, + refresh='true') + + db.session.add_all(returns) + db.session.commit() + return returns + +@pytest.fixture() +def authors2(app,db,esindex): + datas = json_data("data/author2.json") + returns = list() for data in datas: returns.append(Authors( gather_flg=0, @@ -475,4 +718,233 @@ def file_instance(db): size=18, ) db.session.add(file) - db.session.commit() \ No newline at end of file + db.session.commit() + + +# @pytest.fixture() +# def esindex(app2): +# from invenio_search import current_search_client as client +# index_name = app2.config["INDEXER_DEFAULT_INDEX"] +# alias_name = "test-author-alias" + +# with open("tests/data/mappings/author-v1.0.0.json","r") as f: +# mapping = json.load(f) + +# with app2.test_request_context(): +# client.indices.create(index=index_name, body=mapping, ignore=[400]) +# client.indices.put_alias(index=index_name, name=alias_name) + +# yield client + +# with app2.test_request_context(): +# client.indices.delete_alias(index=index_name, name=alias_name) +# client.indices.delete(index=index_name, ignore=[400, 404]) + +from invenio_oauth2server.models import Client + +@pytest.fixture() +def client_model(client_api, users): + """Create client.""" + with db_.session.begin_nested(): + # create resource_owner -> client_1 + client_ = Client( + client_id='client_test_u1c1', + client_secret='client_test_u1c1', + name='client_test_u1c1', + description='', + is_confidential=False, + user=users[0]['obj'], + _redirect_uris='', + _default_scopes='', + ) + db_.session.add(client_) + db_.session.commit() + return client_ + +@pytest.yield_fixture() +def client_api(app): + app.register_blueprint(create_blueprint(app.config['WEKO_AUTHORS_REST_ENDPOINTS'])) + with app.test_client() as client: + yield client + +from invenio_oauth2server.models import Token +from datetime import timedelta +@pytest.fixture() +def create_token_user_noroleuser(client_api, client_model, users): + """Create token.""" + with db_.session.begin_nested(): + token_ = Token( + client=client_model, + user=next((user for user in users if user["id"] == 9), None)['obj'], + token_type='bearer', + access_token='dev_access_create_token_user_noroleuser', + # refresh_token='', + expires=datetime.now() + timedelta(hours=10), + is_personal=True, + is_internal=False, + _scopes="author:create author:delete author:search author:update", + ) + db_.session.add(token_) + db_.session.commit() + return token_ + +@pytest.fixture() +def create_token_user_sysadmin(client_api, client_model, users): + """Create token.""" + print(next((user for user in users if user["id"] == 5), None)['obj']) + with db_.session.begin_nested(): + token_ = Token( + client=client_model, + user=next((user for user in users if user["id"] == 5), None)['obj'], + token_type='bearer', + access_token='dev_access_create_token_user_sysadmin', + # refresh_token='', + expires=datetime.now() + timedelta(hours=10), + is_personal=True, + is_internal=False, + _scopes="author:create author:delete author:search author:update", + ) + db_.session.add(token_) + db_.session.commit() + return token_ + +@pytest.fixture() +def create_token_user_sysadmin_without_scope(client_api, client_model, users): + """Create token.""" + print(next((user for user in users if user["id"] == 5), None)['obj']) + with db_.session.begin_nested(): + token_ = Token( + client=client_model, + user=next((user for user in users if user["id"] == 5), None)['obj'], + token_type='bearer', + access_token='dev_access_create_token_user_sysadmin_without_scope', + # refresh_token='', + expires=datetime.now() + timedelta(hours=10), + is_personal=True, + is_internal=False, + _scopes="", + ) + db_.session.add(token_) + db_.session.commit() + return token_ + +@pytest.fixture() +def json_headers(): + """JSON headers.""" + return [('Content-Type', 'application/json'), + ('Accept', 'application/json')] + +from copy import deepcopy +def fill_oauth2_headers(json_headers, token): + """Create authentication headers (with a valid oauth2 token).""" + headers = deepcopy(json_headers) + headers.append( + ('Authorization', 'Bearer {0}'.format(token.access_token)) + ) + return headers + +@pytest.fixture() +def auth_headers_noroleuser(client_api, json_headers, create_token_user_noroleuser): + """Authentication headers (with a valid oauth2 token). + It uses the token associated with the first user. + """ + return fill_oauth2_headers(json_headers, create_token_user_noroleuser) + +@pytest.fixture() +def auth_headers_sysadmin(client_api, json_headers, create_token_user_sysadmin): + """Authentication headers (with a valid oauth2 token). + It uses the token associated with the first user. + """ + return fill_oauth2_headers(json_headers, create_token_user_sysadmin) + +@pytest.fixture() +def auth_headers_sysadmin_without_scope(client_api, json_headers, create_token_user_sysadmin_without_scope): + """Authentication headers (with a valid oauth2 token). + It uses the token associated with the first user. + """ + return fill_oauth2_headers(json_headers, create_token_user_sysadmin_without_scope) + +@pytest.fixture +def author_records_for_test(app, esindex, db): + record_1_data = { + "emailInfo": [{"email": "sample@xxx.co.jp"}], + "authorIdInfo": [ + {"idType": "2", "authorId": "https://orcid.org/##", "authorIdShowFlg": "true"}, + {"idType": "1", "authorId": "1", "authorIdShowFlg": "true"} + ], + "authorNameInfo": [ + {"language": "en", "firstName": "Test_1", "familyName": "User_1", "nameFormat": "familyNmAndNm", "nameShowFlg": "true"} + ], + "affiliationInfo": [{ + "identifierInfo": [{"affiliationId": "https://ror.org/##", "affiliationIdType": "3", "identifierShowFlg": "true"}], + "affiliationNameInfo": [{"affiliationName": "NII", "affiliationNameLang": "en", "affiliationNameShowFlg": "true"}] + }] + } + + record_2_data = { + "emailInfo": [{"email": "sample@xxx.co.jp"}], + "authorIdInfo": [ + {"idType": "2", "authorId": "https://orcid.org/##", "authorIdShowFlg": "true"}, + {"idType": "1", "authorId": "2", "authorIdShowFlg": "true"} + ], + "authorNameInfo": [ + {"language": "en", "firstName": "Test_2", "familyName": "User_2", "nameFormat": "familyNmAndNm", "nameShowFlg": "true"} + ], + "affiliationInfo": [{ + "identifierInfo": [{"affiliationId": "https://ror.org/##", "affiliationIdType": "3", "identifierShowFlg": "true"}], + "affiliationNameInfo": [{"affiliationName": "NII", "affiliationNameLang": "en", "affiliationNameShowFlg": "true"}] + }] + } + + record_3_data = { + "emailInfo": [{"email": "sample@xxx.co.jp"}], + "authorIdInfo": [ + {"idType": "2", "authorId": "https://orcid.org/##", "authorIdShowFlg": "true"}, + {"idType": "1", "authorId": "3", "authorIdShowFlg": "true"} + ], + "authorNameInfo": [ + {"language": "en", "firstName": "Test_3", "familyName": "User_3", "nameFormat": "familyNmAndNm", "nameShowFlg": "true"} + ], + "affiliationInfo": [{ + "identifierInfo": [{"affiliationId": "https://ror.org/##", "affiliationIdType": "3", "identifierShowFlg": "true"}], + "affiliationNameInfo": [{"affiliationName": "NII", "affiliationNameLang": "en", "affiliationNameShowFlg": "true"}] + }] + } + record_4_data = { + "emailInfo": [{"email": "sample@xxx.co.jp"}], + "authorIdInfo": [ + {"idType": "2", "authorId": "https://orcid.org/##", "authorIdShowFlg": "true"}, + {"idType": "1", "authorId": "4", "authorIdShowFlg": "true"} + ], + "authorNameInfo": [ + {"language": "en", "firstName": "Test_3", "familyName": "User_3", "nameFormat": "familyNmAndNm", "nameShowFlg": "true"} + ], + "affiliationInfo": [{ + "identifierInfo": [{"affiliationId": "https://ror.org/##", "affiliationIdType": "3", "identifierShowFlg": "true"}], + "affiliationNameInfo": [{"affiliationName": "NII", "affiliationNameLang": "en", "affiliationNameShowFlg": "true"}] + }] + } + from weko_authors.api import WekoAuthors + record_1 = WekoAuthors.create(record_1_data) + record_2 = WekoAuthors.create(record_2_data) + record_3 = WekoAuthors.create(record_3_data) + record_3 = WekoAuthors.create(record_4_data) + + esindex.indices.refresh(index=app.config['WEKO_AUTHORS_ES_INDEX_NAME']) + result=[] + for i in range(4): + search_results = esindex.search( + index=app.config['WEKO_AUTHORS_ES_INDEX_NAME'], + body={"query": {"term": {"pk_id": i+1}}}, + size=1 + ) + if search_results["hits"]["total"] > 0: + + result.append(search_results["hits"]["hits"][0]["_id"]) + + return { + "1": result[0], + "2": result[1], + "3": result[2], + "4": result[3] + } \ No newline at end of file diff --git a/modules/weko-authors/tests/data/author.json b/modules/weko-authors/tests/data/author.json index 4e511c157b..90a5803b61 100644 --- a/modules/weko-authors/tests/data/author.json +++ b/modules/weko-authors/tests/data/author.json @@ -10,17 +10,13 @@ } ], "identifierInfo": [ - { - "affiliationId": "5678", - "affiliationIdType": "1", - "identifierShowFlg": "true" - } ] } ], "authorIdInfo": [ { "authorId": "1", "authorIdShowFlg": "true", "idType": "1" }, - { "authorId": "1234", "authorIdShowFlg": "true", "idType": "2" } + { "authorId": "1234", "authorIdShowFlg": "true", "idType": "2" }, + { "authorId": "12345", "authorIdShowFlg": "true", "idType": "3" } ], "authorNameInfo": [ { @@ -35,7 +31,7 @@ "emailInfo": [{ "email": "test.taro@test.org" }], "gather_flg": 0, "id": "8e174b76-66f6-46b3-a2cb-2607fdc1e84b", - "is_deleted": "false", + "is_deleted": false, "pk_id": "1" }, { @@ -53,6 +49,11 @@ "affiliationId": "1234", "affiliationIdType": "1", "identifierShowFlg": "true" + }, + { + "affiliationId": "12345", + "affiliationIdType": "2", + "identifierShowFlg": "true" } ] } @@ -74,7 +75,33 @@ "emailInfo": [{ "email": "test.smith@test.org" }], "gather_flg": 0, "id": "12bdc98a-20fb-4455-92ba-ad1131dbfcc8", - "is_deleted": "false", + "is_deleted": false, "pk_id": "2" + }, + { + "authorIdInfo": [ + { "authorId": "123", "authorIdShowFlg": "true", "idType": "" } + ], + "id": "24d4fa17-5d91-4d78-9e27-1a63a1c7b06f", + "pk_id": "3" + }, + { + "affiliationInfo": [ + { + "affiliationNameInfo": [ + {} + ], + "identifierInfo": [ + {} + ] + } + ], + "authorIdInfo": [ + { "authorId": "2", "authorIdShowFlg": "true", "idType": "2" } + ], + "gather_flg": 1, + "id": "1ac8be125-18a0-4521-8392-b920f79fdecc", + "pk_id": "4", + "is_deleted": true } ] diff --git a/modules/weko-authors/tests/data/author2.json b/modules/weko-authors/tests/data/author2.json new file mode 100644 index 0000000000..153055277c --- /dev/null +++ b/modules/weko-authors/tests/data/author2.json @@ -0,0 +1,102 @@ +[ + { + "affiliationInfo": [ + { + "affiliationNameInfo": [ + { + "affiliationName": "", + "affiliationNameLang": "ja", + "affiliationNameShowFlg": "true" + } + ], + "identifierInfo": [ + ] + } + ], + "authorIdInfo": [ + { "authorId": "1", "authorIdShowFlg": "true", "idType": "1" }, + { "authorId": "1234", "authorIdShowFlg": "true", "idType": "2" }, + { "authorId": "12345", "authorIdShowFlg": "true", "idType": "3" } + ], + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "fullName": "テスト 太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true" + } + ], + "emailInfo": [{ "email": "test.taro@test.org" }], + "gather_flg": 0, + "id": "8e174b76-66f6-46b3-a2cb-2607fdc1e84b", + "is_deleted": "false", + "pk_id": "1" + }, + { + "affiliationInfo": [ + { + "affiliationNameInfo": [ + { + "affiliationName": "", + "affiliationNameLang": "ja", + "affiliationNameShowFlg": "true" + } + ], + "identifierInfo": [ + { + "affiliationId": "1234", + "affiliationIdType": "1", + "identifierShowFlg": "true" + }, + { + "affiliationId": "12345", + "affiliationIdType": "2", + "identifierShowFlg": "true" + } + ] + } , + { + "affiliationNameInfo": [ + { + "affiliationName": "", + "affiliationNameLang": "ja", + "affiliationNameShowFlg": "true" + } + ], + "identifierInfo": [ + { + "affiliationId": "1234", + "affiliationIdType": "1", + "identifierShowFlg": "true" + }, + { + "affiliationId": "12345", + "affiliationIdType": "2", + "identifierShowFlg": "true" + } + ] + } + ], + "authorIdInfo": [ + { "authorId": "2", "authorIdShowFlg": "true", "idType": "1" }, + { "authorId": "5678", "authorIdShowFlg": "true", "idType": "2" } + ], + "authorNameInfo": [ + { + "familyName": "test", + "firstName": "smith", + "fullName": "test smith", + "language": "en", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true" + } + ], + "emailInfo": [{ "email": "test.smith@test.org" }], + "gather_flg": 0, + "id": "12bdc98a-20fb-4455-92ba-ad1131dbfcc8", + "is_deleted": "false", + "pk_id": "2" + } +] diff --git a/modules/weko-authors/tests/data/mappings/author-v1.0.0.json b/modules/weko-authors/tests/data/mappings/author-v1.0.0.json new file mode 100644 index 0000000000..c4a133c93f --- /dev/null +++ b/modules/weko-authors/tests/data/mappings/author-v1.0.0.json @@ -0,0 +1,240 @@ +{ + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + }, + "mappings": { + "author-v1.0.0": { + "properties": { + "affiliationInfo": { + "properties": { + "affiliationNameInfo": { + "properties": { + "affiliationName": { + "type": "text", + "index": false + }, + "affiliationNameLang": { + "type": "text" + }, + "affiliationNameShowFlg": { + "type": "boolean" + } + } + }, + "identifierInfo": { + "properties": { + "affiliationId": { + "type": "text" + }, + "affiliationIdType": { + "type": "text", + "index": false + }, + "identifierShowFlg": { + "type": "boolean" + } + } + } + } + }, + "authorIdInfo": { + "properties": { + "authorId": { + "type": "text" + }, + "authorIdShowFlg": { + "type": "boolean" + }, + "idType": { + "type": "text", + "index": false + } + } + }, + "authorNameInfo": { + "properties": { + "familyName": { + "type": "text" + }, + "firstName": { + "type": "text" + }, + "fullName": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + }, + "language": { + "type": "text", + "index": false + }, + "nameFormat": { + "type": "text", + "index": false + }, + "nameShowFlg": { + "type": "boolean" + } + } + }, + "emailInfo": { + "properties": { + "email": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + } + } + } + } + }, + "gather_flg": { + "type": "long" + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "is_deleted": { + "type": "boolean" + }, + "pk_id": { + "type": "keyword" + }, + "query": { + "properties": { + "bool": { + "properties": { + "must": { + "properties": { + "bool": { + "properties": { + "should": { + "properties": { + "bool": { + "properties": { + "must": { + "properties": { + "term": { + "properties": { + "is_deleted": { + "properties": { + "value": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } + } + } + } + } + } + } + } + }, + "term": { + "properties": { + "gather_flg": { + "type": "long" + } + } + } + } + } + } + }, + "match_all": { + "type": "object" + } + } + } + }, + "_meta": { + "creation_date": "1704763206126", + "provided_name": "test-authors", + "max_result_window": "1000000", + "analysis": { + "analyzer": { + "ngram_analyzer": { + "filter": [ + "cjk_width", + "lowercase" + ], + "char_filter": [ + "html_strip" + ], + "type": "custom", + "tokenizer": "ngram_tokenizer" + }, + "ja_analyzer": { + "filter": [ + "kuromoji_baseform", + "kuromoji_part_of_speech", + "cjk_width", + "stop", + "kuromoji_stemmer", + "lowercase" + ], + "tokenizer": "ja_tokenizer" + }, + "paths": { + "tokenizer": "path_hierarchy" + }, + "wk_analyzer": { + "filter": [ + "standard", + "lowercase", + "stop", + "cjk_width" + ], + "char_filter": [ + "html_strip" + ], + "type": "custom", + "tokenizer": "standard" + } + }, + "tokenizer": { + "ja_tokenizer": { + "mode": "search", + "type": "kuromoji_tokenizer" + }, + "ngram_tokenizer": { + "token_chars": [ + "letter", + "digit" + ], + "min_gram": "2", + "type": "nGram", + "max_gram": "3" + } + } + }, + "uuid": "en03boXqSpGbBiCrGN4NIg", + "version": { + "created": "6082399" + } + } + } + } +} \ No newline at end of file diff --git a/modules/weko-authors/tests/data/test_authors/author01.json b/modules/weko-authors/tests/data/test_authors/author01.json new file mode 100644 index 0000000000..6d913f8380 --- /dev/null +++ b/modules/weko-authors/tests/data/test_authors/author01.json @@ -0,0 +1,51 @@ +{ + "id": "", + "pk_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "fullName": "テスト 太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true" + } + ], + "authorIdInfo": [ + { + "idType": "1", + "authorId": "1", + "authorIdShowFlg": "true" + }, + { + "idType": "2", + "authorId": "q", + "authorIdShowFlg": "true" + } + ], + "emailInfo": [ + { + "email": "test.taro@test.org" + } + ], + "affiliationInfo": [ + { + "identifierInfo": [ + { + "affiliationIdType": "1", + "affiliationId": "aaaaaa", + "identifierShowFlg": "true" + } + ], + "affiliationNameInfo": [ + { + "affiliationName": "aaaaaaaaaaaa", + "affiliationNameLang": "ja", + "affiliationNameShowFlg": "true" + } + ] + } + ], + "gather_flg": 0, + "is_deleted": "false" +} \ No newline at end of file diff --git a/modules/weko-authors/tests/data/test_authors/author02.json b/modules/weko-authors/tests/data/test_authors/author02.json new file mode 100644 index 0000000000..99b97f1ade --- /dev/null +++ b/modules/weko-authors/tests/data/test_authors/author02.json @@ -0,0 +1,51 @@ +{ + "id": "", + "pk_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "fullName": "テスト 太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true" + } + ], + "authorIdInfo": [ + { + "idType": "1", + "authorId": "1", + "authorIdShowFlg": "true" + }, + { + "idType": "2", + "authorId": "q", + "authorIdShowFlg": "true" + } + ], + "emailInfo": [ + { + "email": "test.taro@test.org" + } + ], + "affiliationInfo": [ + { + "identifierInfo": [ + { + "affiliationIdType": "1", + "affiliationId": "aaaaaa", + "identifierShowFlg": "true" + } + ], + "affiliationNameInfo": [ + { + "affiliationName": "aaaaaaaaaaaa", + "affiliationNameLang": "ja", + "affiliationNameShowFlg": "true" + } + ] + } + ], + "gather_flg": 1, + "is_deleted": "false" +} \ No newline at end of file diff --git a/modules/weko-authors/tests/data/test_authors/author03.json b/modules/weko-authors/tests/data/test_authors/author03.json new file mode 100644 index 0000000000..76c59979b4 --- /dev/null +++ b/modules/weko-authors/tests/data/test_authors/author03.json @@ -0,0 +1,51 @@ +{ + "id": "", + "pk_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "fullName": "テスト 太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true" + } + ], + "authorIdInfo": [ + { + "idType": "1", + "authorId": "1", + "authorIdShowFlg": "true" + }, + { + "idType": "2", + "authorId": "q", + "authorIdShowFlg": "true" + } + ], + "emailInfo": [ + { + "email": "test.taro@test.org" + } + ], + "affiliationInfo": [ + { + "identifierInfo": [ + { + "affiliationIdType": "1", + "affiliationId": "aaaaaa", + "identifierShowFlg": "true" + } + ], + "affiliationNameInfo": [ + { + "affiliationName": "aaaaaaaaaaaa", + "affiliationNameLang": "ja", + "affiliationNameShowFlg": "true" + } + ] + } + ], + "gather_flg": 0, + "is_deleted": "true" +} \ No newline at end of file diff --git a/modules/weko-authors/tests/test_admin.py b/modules/weko-authors/tests/test_admin.py index 209168a5fc..5bd42a40f1 100644 --- a/modules/weko-authors/tests/test_admin.py +++ b/modules/weko-authors/tests/test_admin.py @@ -7,6 +7,12 @@ from invenio_accounts.testutils import login_user_via_session from invenio_cache import current_cache from invenio_files_rest.models import FileInstance +from weko_workflow.utils import delete_cache_data +from unittest.mock import MagicMock, mock_open +from datetime import datetime +from flask import current_app +from weko_authors.admin import ImportView +from weko_authors.tasks import import_author, import_id_prefix, import_affiliation_id def assert_role(response,is_permission,status_code=403): if is_permission: @@ -247,11 +253,7 @@ def test_download(self,client,db,users,mocker): mock_send = mocker.patch("weko_authors.admin.FileInstance.send_file",return_value=make_response()) res = client.get(url) assert res.status_code == 200 - mock_send.assert_called_with( - "Creator_export_all.tsv", - mimetype="application/octet-stream", - as_attachment=True - ) + mock_send.assert_called() # def check_status(self): @@ -296,14 +298,14 @@ def failed(self): current_cache.set("weko_authors_exported_url",{"key":"authors_exported_url","file_uri":"test_file.txt"}) mocker.patch("weko_authors.admin.export_all.AsyncResult",return_value=MockAsyncResult("test_id","SUCCESS","result")) res = client.get(url) - test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'key': 'authors_exported_url'}} + test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'filename': '', 'key': 'authors_exported_url'}} assert json.loads(res.data)==test - # not task.result + # # not task.result current_cache.set("weko_authors_export_status",{"key":"authors_export_status","task_id":"test_task"}) mocker.patch("weko_authors.admin.export_all.AsyncResult",return_value=MockAsyncResult("test_id","SUCCESS",{})) res = client.get(url) - test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'error': 'export_fail', 'key': 'authors_exported_url'}} + test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'error': 'export_fail', 'filename': '', 'key': 'authors_exported_url'}} assert json.loads(res.data)==test # not task is success,failed,revoked @@ -311,22 +313,43 @@ def failed(self): current_cache.set("weko_authors_exported_url",{"key":"authors_exported_url","file_uri":"test_file.txt"}) mocker.patch("weko_authors.admin.export_all.AsyncResult",return_value=MockAsyncResult("test_id","STARTED",{})) res = client.get(url) - test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'key': 'authors_export_status', "task_id": "test_task"}} + test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'filename': '', 'key': 'authors_export_status', "task_id": "test_task"}} assert json.loads(res.data) == test # not exist get_export_status current_cache.delete("weko_authors_export_status") current_cache.set("weko_authors_exported_url",{"key":"authors_exported_url","file_uri":"test_file.txt"}) res = client.get(url) - test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'key': 'authors_exported_url'}} + test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'filename': '', 'key': 'authors_exported_url'}} assert json.loads(res.data) == test # exist weko_authors_export_status,not exist weko_authors_export_status[task_id] current_cache.set("weko_authors_export_status",{"key":"authors_export_status"}) res = client.get(url) - test = {'code': 200, 'data': {'download_link': '', 'key': 'authors_export_status'}} + test = {'code': 200, 'data': {'download_link': '', 'filename': '', 'key': 'authors_export_status'}} assert json.loads(res.data) == test - + + # not get_export_status + current_cache.set("weko_authors_export_stop_point",{"key":"authors_export_stop_point"}) + current_cache.set("weko_authors_export_status", []) + current_cache.set("weko_authors_exported_url",{"key":"authors_exported_url","file_uri":"test_file.txt"}) + mocker.patch("weko_authors.admin.export_all.AsyncResult",return_value=MockAsyncResult("test_id","SUCCESS","result")) + res = client.get(url) + test = {'code': 200, 'data': {'download_link':'http://app/admin/authors/export/download/Creator_export_all','filename':'','key':'authors_exported_url','stop_point':{'key':'authors_export_stop_point'}}} + assert json.loads(res.data)==test + delete_cache_data("weko_authors_export_stop_point") + + # exsit FileInstance.get_by_uri + current_cache.set("weko_authors_export_status",{"key":"authors_export_status","task_id":"test_task"}) + current_cache.set("weko_authors_exported_url",{"key":"authors_exported_url","file_uri":"test_file.txt"}) + mocker.patch("weko_authors.admin.export_all.AsyncResult",return_value=MockAsyncResult("test_id","SUCCESS","result")) + mock_file_instance = MagicMock(spec=FileInstance) + mock_file_instance.updated = datetime(2020, 8, 28, 8, 28) + mocker.patch("weko_authors.admin.FileInstance.get_by_uri",return_value=mock_file_instance) + res = client.get(url) + test = {'code': 200, 'data': {'download_link': 'http://app/admin/authors/export/download/Creator_export_all', 'filename': '_202008280828.tsv', 'key': 'authors_exported_url'}} + assert json.loads(res.data)==test + # def export(self): # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestExportView::test_export_acl_guest -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -359,15 +382,18 @@ class MockTask: # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestExportView::test_export -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_export(self,client,users,mocker): - login_user_via_session(client=client, email=users[0]['email']) url = url_for('authors/export.export') mocker.patch("weko_authors.admin.set_export_status") class MockTask: id = "test_id" - mocker.patch("weko_authors.admin.export_all.delay",return_value=MockTask) - res = client.post(url) + mocker.patch("weko_authors.admin.export_all.delay",return_value=MockTask) + login_user_via_session(client=client, email=users[0]['email']) + data = {"isTarget":None} + res = client.post(url, data=json.dumps(data), content_type='application/json') + assert json.loads(res.data) == {"code":200,"data":{"task_id":"test_id"}} + data = {"isTarget":"author_db"} + res = client.post(url, data=json.dumps(data), content_type='application/json') assert json.loads(res.data) == {"code":200,"data":{"task_id":"test_id"}} - # def cancel(self): # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestExportView::test_cancel_acl_guest -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -400,10 +426,18 @@ def test_cancel(self,client,users,mocker): url = url_for('authors/export.cancel') mocker.patch("weko_authors.admin.revoke") current_cache.set("weko_authors_export_status",{"key":"authors_export_status","task_id":"test_task"}) - + current_cache.set("weko_authors_export_stop_point",{"key":"authors_export_stop_point"}) + current_cache.set("weko_authors_export_temp_file_path_key",{"key":"authors_export_temp_file_path_key"}) + mocker.patch("os.remove") res = client.post(url) assert json.loads(res.data) == {"code":200,"data":{"status":"success"}} + # not temp_file_path + current_cache.set("weko_authors_export_status",None) + current_cache.set("weko_authors_export_stop_point",{"key":"authors_export_stop_point"}) + res = client.post(url) + assert json.loads(res.data) == {"code":200,"data":{"status":"fail"}} + # not exist status res = client.post(url) assert json.loads(res.data) == {"code":200,"data":{"status":"fail"}} @@ -413,6 +447,18 @@ def test_cancel(self,client,users,mocker): with patch("weko_authors.admin.get_export_status",side_effect=Exception("test_error")): res = client.post(url) assert json.loads(res.data) == test + + # def resume(self): + # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestExportView::test_resume -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_resume(self,client,users,mocker): + url = url_for('authors/export.resume') + mocker.patch("weko_authors.admin.set_export_status") + class MockTask: + id = "test_id" + mocker.patch("weko_authors.admin.export_all.delay",return_value=MockTask) + login_user_via_session(client=client, email=users[0]['email']) + res = client.post(url) + assert json.loads(res.data) == {"code":200,"data":{"task_id":"test_id"}} # class ImportView(BaseView): @@ -521,14 +567,42 @@ def test_check_import_file(self,client,users,mocker): # not exist json res = client.post(url,json={}) - assert json.loads(res.data) == {"code":1,"error":None,"list_import_data":[]} + assert json.loads(res.data) == {"code":1,"error":None,"list_import_data":[],"counts":0,"max_page":1} - # exist json - mocker.patch("weko_authors.admin.check_import_data",return_value={"list_import_data":["test_import_data"]}) - res = client.post(url,json={"filename":"test_file.txt","file":"test1,test2"}) - assert json.loads(res.data) == {"code":1,"error":None,"list_import_data":["test_import_data"]} + # target == "id_prefix" + mocker.patch("weko_authors.admin.check_import_data_for_prefix",return_value={"list_import_data":["test_import_data"]}) + res = client.post(url,json={"file_name":"test_file.txt","file":"test1,test2","target":"id_prefix"}) + assert json.loads(res.data) == {"code":1,"error":None,"list_import_data":["test_import_data"],"counts":0,"max_page":1} + + # target == "author_db" + mocker.patch("weko_authors.admin.check_import_data_for_prefix",return_value={"list_import_data":["test_import_data"]}) + mocker.patch("weko_authors.admin.check_import_data", return_value={"error":"Internal server error"}) + current_cache.set("authors_import_band_check_user_file_path",{"key":"authors_import_band_check_user_file_path"}) + mocker.patch("os.remove") + res = client.post(url,json={"file_name":"test_file.txt","file":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKAQMAAAC3/F3+AAAAA3NCSVQICAjb4U/gAAAABlBMVEUjHyD///9mY0coAAAACXBIWXMAAAsSAAALEgHS3X78AAAAFnRFWHRDcmVhdGlvbiBUaW1lADAzLzIzLzEysFVRHgAAABx0RVh0U29mdHdhcmUAQWRvYmUgRmlyZXdvcmtzIENTNXG14zYAAAAWSURBVAiZY/h/gAGIPoPRATCCsMHiAPy6EMmRpJhhAAAAAElFTkSuQmCC","target":"author_db"}) + assert json.loads(res.data) == {"code":1,"error":"Internal server error","list_import_data":None,"counts":None,"max_page":None} + # not band_file_path + mocker.patch("weko_authors.admin.check_import_data_for_prefix",return_value={"list_import_data":["test_import_data"]}) + mocker.patch("weko_authors.admin.check_import_data", return_value={"error":"Internal server error"}) + res = client.post(url,json={"file_name":"test_file.txt","file":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKAQMAAAC3/F3+AAAAA3NCSVQICAjb4U/gAAAABlBMVEUjHyD///9mY0coAAAACXBIWXMAAAsSAAALEgHS3X78AAAAFnRFWHRDcmVhdGlvbiBUaW1lADAzLzIzLzEysFVRHgAAABx0RVh0U29mdHdhcmUAQWRvYmUgRmlyZXdvcmtzIENTNXG14zYAAAAWSURBVAiZY/h/gAGIPoPRATCCsMHiAPy6EMmRpJhhAAAAAElFTkSuQmCC","target":"author_db"}) + assert json.loads(res.data) == {"code":1,"error":"Internal server error","list_import_data":None,"counts":None,"max_page":None} + # target == "dummy" + mocker.patch("weko_authors.admin.check_import_data_for_prefix",return_value={"list_import_data":["test_import_data"]}) + res = client.post(url,json={"file_name":"test_file.txt","file":"test1,test2","target":"dummy"}) + assert json.loads(res.data) == {"code":1,"error":None,"list_import_data":[],"counts":0,"max_page":1} + + # Exception + mock_logger = MagicMock() + current_app.logger = mock_logger + mocker.patch("weko_authors.admin.check_import_data_for_prefix",return_value={"list_import_data":["test_import_data"]}) + mocker.patch("weko_authors.admin.check_import_data", return_value={"error":"Internal server error"}) + current_cache.set("authors_import_band_check_user_file_path",{"key":"authors_import_band_check_user_file_path"}) + mocker.patch("os.remove", side_effect=FileNotFoundError) + res = client.post(url,json={"file_name":"test_file.txt","file":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKAQMAAAC3/F3+AAAAA3NCSVQICAjb4U/gAAAABlBMVEUjHyD///9mY0coAAAACXBIWXMAAAsSAAALEgHS3X78AAAAFnRFWHRDcmVhdGlvbiBUaW1lADAzLzIzLzEysFVRHgAAABx0RVh0U29mdHdhcmUAQWRvYmUgRmlyZXdvcmtzIENTNXG14zYAAAAWSURBVAiZY/h/gAGIPoPRATCCsMHiAPy6EMmRpJhhAAAAAElFTkSuQmCC","target":"author_db"}) + mock_logger.error.assert_called_once_with("Error deleting {'key': 'authors_import_band_check_user_file_path'}: ") + # def import_authors(self) -> jsonify: # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestImportView::test_import_authors_acl_guest -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_import_authors_acl_guest(self,client): @@ -558,14 +632,6 @@ def test_import_authors_acl(self,client,users,users_index,is_permission,mocker): def test_import_authors(self,client,users,mocker): login_user_via_session(client=client, email=users[0]['email']) url = url_for('authors/import.import_authors') - - # not is_available - with patch("weko_authors.admin.check_is_import_available",return_value={"is_available":False}): - res = client.post(url) - assert json.loads(res.data) == {"is_available":False} - - mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":True}) - class MockTaskGroup: def __init__(self): self.id = 1 @@ -573,25 +639,223 @@ def save(self): pass @property def children(self): - return [self.MockTask(id) for id in range(4)] + return [self.MockTask(id) for id in range(2)] class MockTask: def __init__(self,id): self.task_id = id - data = {"records":[ - {"pk_id":"test_id0"},{"pk_id":"test_id1"},{"pk_id":"test_id2"},{"pk_id":"test_id3"} - ]} + # not is_available + mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":False}) + res = client.post(url) + assert json.loads(res.data) == {"is_available":False} + + # is_target == "id_prefix" + data = { + "isTarget": "id_prefix", + "max_page": 1, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":True}) + mocker.patch("weko_authors.admin.group.apply_async",return_value=MockTaskGroup()) + res = client.post(url, data=json.dumps(data), content_type='application/json') + test = { + "status": "success", + "count": 0, + "data": { + "group_task_id": 1, + "tasks": [ + {"task_id": 0, "scheme": "WEKO", "name": "name0", "status": "PENDING"}, + {"task_id": 1, "scheme": "GRID", "name": "name1", "status": "PENDING"}, + ], + }, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + assert json.loads(res.data) == test + + # is_target == "affiliation_id" + data = { + "isTarget": "affiliation_id", + "max_page": 1, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":True}) mocker.patch("weko_authors.admin.group.apply_async",return_value=MockTaskGroup()) - res = client.post(url,json=data) + res = client.post(url, data=json.dumps(data), content_type='application/json') + test = { + "status": "success", + "count": 0, + "data": { + "group_task_id": 1, + "tasks": [ + {"task_id": 0, "scheme": "WEKO", "name": "name0", "status": "PENDING"}, + {"task_id": 1, "scheme": "GRID", "name": "name1", "status": "PENDING"}, + ], + }, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + assert json.loads(res.data) == test + + # is_target == "author_db" + data = { + "isTarget": "author_db", + "max_page": 1, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":True}) + current_cache.set("cache_result_over_max_file_path_key",{"key":"cache_result_over_max_file_path_key"}) + mocker.patch("os.remove") + current_cache.set("authors_import_result_file_path",{"key":"authors_import_result_file_path"}) + current_cache.set("result_summary_key",{"key":"result_summary_key"}) + mocker.patch("weko_authors.admin.prepare_import_data",return_value=([ + {"pk_id": "test_id0", "current_weko_id": "1000", "weko_id": "1000"}, + {"pk_id": "test_id1", "current_weko_id": "1001", "weko_id": "1001"}, + ], 1, 1)) + mocker.patch("weko_authors.admin.group.apply_async",return_value=MockTaskGroup()) + res = client.post(url, data=json.dumps(data), content_type='application/json') test = { - "status":"success", - "data":{ - "group_task_id":1, - "tasks":[{"task_id":0,"record_id":"test_id0","status":"PENDING"},{"task_id":1,"record_id":"test_id1","status":"PENDING"},{"task_id":2,"record_id":"test_id2","status":"PENDING"},{"task_id":3,"record_id":"test_id3","status":"PENDING"}] - } + "status": "success", + "count": 1, + "data": { + "group_task_id": 1, + "tasks": [ + { + "task_id": 0, + "record_id": "test_id0", + "previous_weko_id": "1000", + "new_weko_id": "1000", + "status": "PENDING", + }, + { + "task_id": 1, + "record_id": "test_id1", + "previous_weko_id": "1001", + "new_weko_id": "1001", + "status": "PENDING", + }, + ], + }, + "records": [ + {"pk_id": "test_id0", "current_weko_id": "1000", "weko_id": "1000"}, + {"pk_id": "test_id1", "current_weko_id": "1001", "weko_id": "1001"}, + ], } assert json.loads(res.data) == test + + # is_target == "dummy" + data = { + "isTarget": "dummy", + "max_page": 1, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":True}) + res = client.post(url, data=json.dumps(data), content_type='application/json') + assert res.status_code == 200 + assert json.loads(res.data) == {'status': 'fail', 'message': 'Invalid target'} + + # result_over_max_file_path, result_file_path, result_summary is None and count > current_app.config.get is true + data = { + "isTarget": "author_db", + "max_page": 1, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":True}) + current_cache.set("cache_result_over_max_file_path_key",None) + current_cache.set("authors_import_result_file_path",None) + current_cache.set("result_summary_key",None) + mocker.patch("weko_authors.admin.prepare_import_data",return_value=([ + {"pk_id": "test_id0", "current_weko_id": "1000", "weko_id": "1000"}, + {"pk_id": "test_id1", "current_weko_id": "1001", "weko_id": "1001"}, + ], 1, 2000)) + mocker.patch("weko_authors.admin.group.apply_async",return_value=MockTaskGroup()) + mock_task = MagicMock() + mock_task.id = 'mocked_task_id' + mocker.patch("weko_authors.admin.import_author_over_max.delay",return_value=mock_task) + mocker.patch("weko_authors.admin.update_cache_data",return_value=None) + res = client.post(url, data=json.dumps(data), content_type='application/json') + test = { + "status": "success", + "count": 2000, + "data": { + "group_task_id": 1, + "tasks": [ + { + "task_id": 0, + "record_id": "test_id0", + "previous_weko_id": "1000", + "new_weko_id": "1000", + "status": "PENDING", + }, + { + "task_id": 1, + "record_id": "test_id1", + "previous_weko_id": "1001", + "new_weko_id": "1001", + "status": "PENDING", + }, + ], + }, + "records": [ + {"pk_id": "test_id0", "current_weko_id": "1000", "weko_id": "1000"}, + {"pk_id": "test_id1", "current_weko_id": "1001", "weko_id": "1001"}, + ], + } + assert json.loads(res.data) == test + + # Exception (result_over_max_file_path is true) + data = { + "isTarget": "author_db", + "max_page": 1, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + mock_logger = MagicMock() + current_app.logger = mock_logger + mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":True}) + current_cache.set("cache_result_over_max_file_path_key",{"key":"cache_result_over_max_file_path_key"}) + mocker.patch("os.remove", side_effect=FileNotFoundError) + client.post(url, data=json.dumps(data), content_type='application/json') + mock_logger.error.assert_called_once_with("Error deleting {'key': 'cache_result_over_max_file_path_key'}: ") + + # Exception (result_file_path is true) + data = { + "isTarget": "author_db", + "max_page": 1, + "records": [ + {"pk_id": "test_id0", "scheme": "WEKO", "name": "name0"}, + {"pk_id": "test_id1", "scheme": "GRID", "name": "name1"}, + ], + } + mock_logger = MagicMock() + current_app.logger = mock_logger + mocker.patch("weko_authors.admin.check_is_import_available",return_value={"is_available":True}) + current_cache.set("authors_import_result_file_path",{"key":"authors_import_result_file_path"}) + mocker.patch("os.remove", side_effect=FileNotFoundError) + client.post(url, data=json.dumps(data), content_type='application/json') + mock_logger.error.assert_called_once_with("Error deleting {'key': 'authors_import_result_file_path'}: ") # def check_import_status(self): # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestImportView::test_check_import_status_acl_guest -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -637,7 +901,7 @@ def __init__(self,id,start,end,status,error): tasks = list() tasks.append(MockAsyncResult(1,"2022-10-01 01:02:03","2022-10-01 02:03:04","SUCCESS","not_error")) tasks.append(MockAsyncResult(2,None,None,None,None)) - mocker.patch("weko_authors.admin.import_author.AsyncResult",side_effect=lambda x:[task for task in tasks if task.id == x][0]) + mocker.patch("weko_authors.admin.ImportView.get_task",side_effect=lambda target, task_id:[task for task in tasks if task.id == task_id][0]) # not exist data data = {} @@ -645,11 +909,223 @@ def __init__(self,id,start,end,status,error): assert res.status_code == 200 assert json.loads(res.data) == [] - data = {"tasks":[1,2]} + # isTarget == "dummy" + data = {"isTarget": "dummy"} + res = client.post(url,json=data) + assert res.status_code == 200 + assert json.loads(res.data) == [] + + # isTarget == "author_db" + data = {"isTarget": "author_db", "tasks": [1, 2]} + test = { + "over_max": { + "error_id": "not_error", + "status": "SUCCESS", + "task_id": {"key": "authors_import_over_max_task"}, + }, + "summary": {"failure_count": 5, "success_count": 6}, + "tasks": [ + { + "task_id": 1, + "start_date": "2022-10-01 01:02:03", + "end_date": "2022-10-01 02:03:04", + "status": "SUCCESS", + "error_id": "not_error", + }, + { + "task_id": 2, + "start_date": "", + "end_date": "", + "status": "PENDING", + "error_id": None, + }, + ], + } + current_cache.set("authors_import_over_max_task",{"key":"authors_import_over_max_task"}) + current_cache.set("result_summary_key",{"success_count":5,"failure_count":5}) + mocker.patch("weko_authors.admin.import_author_over_max.AsyncResult",side_effect=lambda x:[task for task in tasks][0]) + res = client.post(url,json=data) + assert res.status_code == 200 + assert json.loads(res.data) == test + + # isTarget == "id_prefix" + data = {"isTarget": "id_prefix", "tasks": [1,2]} test = [ - {"task_id":1,"start_date":"2022-10-01 01:02:03","end_date":"2022-10-01 02:03:04","status":"SUCCESS","error_id":"not_error"}, - {"task_id":2,"start_date":"","end_date":"","status":"PENDING","error_id":None} + { + "task_id": 1, + "start_date": "2022-10-01 01:02:03", + "end_date": "2022-10-01 02:03:04", + "status": "FAILURE", + "error_id": "error_id", + }, + { + "task_id": 2, + "start_date": "", + "end_date": "", + "status": "RUNNING", + "error_id": "not_error", + }, ] + failure_tasks = list() + failure_tasks.append(MockAsyncResult(1,"2022-10-01 01:02:03","2022-10-01 02:03:04","FAILURE","error_id")) + failure_tasks.append(MockAsyncResult(2,None,None,"RUNNING","not_error")) + mocker.patch("weko_authors.admin.ImportView.get_task",side_effect=lambda target, task_id:[task for task in failure_tasks if task.id == task_id][0]) + res = client.post(url,json=data) + assert res.status_code == 200 + assert json.loads(res.data) == test + + # over_max_task, summary is None + data = {"isTarget": "author_db", "tasks": [1, 2]} + test = { + "summary": {"failure_count": 0, "success_count": 1}, + "tasks": [ + { + "task_id": 1, + "start_date": "2022-10-01 01:02:03", + "end_date": "2022-10-01 02:03:04", + "status": "SUCCESS", + "error_id": "not_error", + }, + { + "task_id": 2, + "start_date": "", + "end_date": "", + "status": "PENDING", + "error_id": None, + }, + ], + } + mocker.patch("weko_authors.admin.ImportView.get_task",side_effect=lambda target, task_id:[task for task in tasks if task.id == task_id][0]) + current_cache.set("authors_import_over_max_task",None) + current_cache.set("result_summary_key",None) res = client.post(url,json=data) assert res.status_code == 200 - assert json.loads(res.data) == test \ No newline at end of file + assert json.loads(res.data) == test + + # if task.result and task.result.get('status') is false + data = {"isTarget": "author_db", "tasks": [1, 2]} + test = { + "over_max": { + "error_id": None, + "status": "PENDING", + "task_id": {"key": "authors_import_over_max_task"}, + }, + "summary": {"failure_count": 5, "success_count": 6}, + "tasks": [ + { + "task_id": 1, + "start_date": "2022-10-01 01:02:03", + "end_date": "2022-10-01 02:03:04", + "status": "SUCCESS", + "error_id": "not_error", + }, + { + "task_id": 2, + "start_date": "", + "end_date": "", + "status": "PENDING", + "error_id": None, + }, + ], + } + current_cache.set("authors_import_over_max_task",{"key":"authors_import_over_max_task"}) + current_cache.set("result_summary_key",{"success_count":5,"failure_count":5}) + mocker.patch("weko_authors.admin.import_author_over_max.AsyncResult",side_effect=lambda x:[task for task in tasks][1]) + res = client.post(url,json=data) + assert res.status_code == 200 + assert json.loads(res.data) == test + + # def check_pagination(self): + # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestImportView::test_check_pagination -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_check_pagination(self,client,users): + login_user_via_session(client=client, email=users[0]['email']) + url = url_for('authors/import.check_pagination') + args = {"page_number":1} + mock_data = {"key": "value"} + with patch("builtins.open", mock_open(read_data=json.dumps(mock_data))) as mock_file, \ + patch("json.load", return_value=mock_data) as mock_json: + with open("somefile.json", "r", encoding="utf-8-sig") as check_part_file: + res = client.get(url,query_string=args) + assert res.status_code == 200 + assert json.loads(res.data) == mock_data + + # def check_file_download(self): + # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestImportView::test_check_file_download -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_check_file_download(self,client,users,mocker): + current_cache.delete("authors_import_band_check_user_file_path") + login_user_via_session(client=client, email=users[0]['email']) + url = url_for('authors/import.check_file_download') + data = {"max_page":1} + mocker.patch("weko_authors.admin.band_check_file_for_user",return_value="test_file_path") + mock_send = mocker.patch("weko_authors.admin.send_file",return_value=make_response()) + client.post(url, data=json.dumps(data), content_type='application/json') + mock_send.assert_called_with( + "test_file_path", + as_attachment=True + ) + mocker.patch("weko_authors.admin.band_check_file_for_user",return_value="test_file_path") + mock_send_file = mocker.patch("weko_authors.admin.send_file", side_effect=Exception("File not found")) + current_cache.set("authors_import_band_check_user_file_path",{"key":"authors_import_band_check_user_file_path"}) + res = client.post(url, data=json.dumps(data), content_type='application/json') + mock_send_file.assert_called_once() + assert res.status_code == 500 + assert json.loads(res.data) == {"msg":"Failed"} + + # def get_task(self, target, task_id): + # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestImportView::test_get_task -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_get_task(self,client,users,mocker): + def mock_async_result(): + mock_result = MagicMock() + mock_result.status = 'SUCCESS' + return mock_result + + view = ImportView() + import_id_prefix.AsyncResult = MagicMock(return_value=mock_async_result) + import_affiliation_id.AsyncResult = MagicMock(return_value=mock_async_result) + import_author.AsyncResult = MagicMock(return_value=mock_async_result) + + task = view.get_task('id_prefix', 100) + assert task == mock_async_result + import_id_prefix.AsyncResult.assert_called_once_with(100) + task = view.get_task('affiliation_id', 200) + assert task == mock_async_result + import_affiliation_id.AsyncResult.assert_called_once_with(200) + task = view.get_task('author_db', 300) + assert task == mock_async_result + import_author.AsyncResult.assert_called_once_with(300) + task = view.get_task('invalid_target', 100) + assert task is None + + # result_file_download(self): + # .tox/c1/bin/pytest --cov=weko_authors tests/test_admin.py::TestImportView::test_result_file_download -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_result_file_download(self,client,users,mocker): + login_user_via_session(client=client, email=users[0]['email']) + url = url_for('authors/import.result_file_download') + data = {"max_page":1} + + current_cache.set("authors_import_result_file_path",{"key":"authors_import_result_file_path"}) + mock_send = mocker.patch("weko_authors.admin.send_file",return_value=make_response()) + client.post(url, data=json.dumps(data), content_type='application/json') + mock_send.assert_called_with( + {"key":"authors_import_result_file_path"}, + as_attachment=True + ) + current_cache.delete("authors_import_result_file_path") + mocker.patch("weko_authors.admin.create_result_file_for_user",return_value="test_file_path") + mock_send = mocker.patch("weko_authors.admin.send_file",return_value=make_response()) + client.post(url, data=json.dumps(data), content_type='application/json') + mock_send.assert_called_with( + "test_file_path", + as_attachment=True + ) + mocker.patch("weko_authors.admin.create_result_file_for_user",return_value=None) + res = client.post(url, data=json.dumps(data), content_type='application/json') + assert res.status_code == 200 + assert json.loads(res.data) == {"Result": "Dont need to create result file"} + + mocker.patch("weko_authors.admin.create_result_file_for_user",return_value="test_file_path") + mock_send_file = mocker.patch("weko_authors.admin.send_file", side_effect=Exception("File not found")) + res = client.post(url, data=json.dumps(data), content_type='application/json') + mock_send_file.assert_called_once() + assert res.status_code == 500 + assert json.loads(res.data) == {"msg":"Failed"} diff --git a/modules/weko-authors/tests/test_api.py b/modules/weko-authors/tests/test_api.py index 7271406dda..37ac8d60b1 100644 --- a/modules/weko-authors/tests/test_api.py +++ b/modules/weko-authors/tests/test_api.py @@ -1,14 +1,19 @@ -from mock import patch +from mock import patch, MagicMock import json import uuid import pytest +import logging +from _pytest.logging import LogCaptureFixture +from logging import INFO, ERROR from flask import current_app from elasticsearch.exceptions import NotFoundError from invenio_indexer.api import RecordIndexer from invenio_search import current_search_client +from mock import patch, MagicMock + from weko_authors.api import WekoAuthors -from weko_authors.models import Authors, AuthorsPrefixSettings +from weko_authors.models import Authors, AuthorsPrefixSettings, AuthorsAffiliationSettings from weko_authors.config import WEKO_AUTHORS_FILE_MAPPING # .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -17,13 +22,13 @@ class MockClient(): def __init__(self): self.index_ = self.MockIndex() self.return_value="" - def index(self,index,doc_type,body): + def index(self,index,doc_type,id,body): return self.index_ def delete(self,index,doc_type,id): pass def set_return(self,value): self.return_value=value - def search(self,index,doc_type,body): + def search(self,index,doc_type=None,body=None): return self.return_value def update(self,index,doc_type,id,body): pass @@ -52,7 +57,7 @@ def test_create(self,app,db,esindex, mocker): WekoAuthors.create(data) db.session.commit() author = Authors.query.filter_by(id=id).one() - test = {"authorIdInfo": [{"authorId": "1", "authorIdShowFlg": "true", "idType": "1"}], "gather_flg": 0, "id": 1, "pk_id": "1"} + test = {"authorIdInfo": [], "gather_flg": 0, "id": str(es_id), "pk_id": "1"} assert author assert author.json == test res = current_search_client.get(index=current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"],doc_type=current_app.config['WEKO_AUTHORS_ES_DOC_TYPE'],id=str(es_id)) @@ -79,6 +84,7 @@ def test_create(self,app,db,esindex, mocker): is_es=True )],indirect=['base_app']) def test_update(self,app,db,esindex,create_author,mocker): + mocker.patch("weko_deposit.tasks.update_items_by_authorInfo.delay") test_data = { "authorNameInfo": [{"familyName": "テスト","firstName": "ハナコ","fullName": "","language": "ja-Kana","nameFormat": "familyNmAndNm","nameShowFlg": "true"}], "authorIdInfo": [{"idType": "2","authorId": "01234","authorIdShowFlg": "true"}], @@ -120,7 +126,6 @@ def test_update(self,app,db,esindex,create_author,mocker): author_id=2 es_id = create_author(json.loads(json.dumps(test_data)), author_id) with patch("weko_authors.api.db.session.merge",side_effect=Exception("test_error")): - with pytest.raises(Exception): WekoAuthors.update(author_id,data) db.session.rollback() @@ -191,7 +196,103 @@ def test_update(self,app,db,esindex,create_author,mocker): assert Authors.query.filter_by(id=author_id).one().is_deleted==True res = current_search_client.get(index=current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"],doc_type=current_app.config['WEKO_AUTHORS_ES_DOC_TYPE'],id=str(es_id)) assert res["_source"]["is_deleted"] == True - + + # es_author['hits']['hits'][0].get('_id') == es_id is true + test_data = { + "authorNameInfo": [{"familyName": "テスト","firstName": "ハナコ","fullName": "","language": "ja-Kana","nameFormat": "familyNmAndNm","nameShowFlg": "true"}], + "authorIdInfo": [{"idType": "2","authorId": "01234","authorIdShowFlg": "true"}], + "emailInfo": [{"email": "example@com"}], + "is_deleted":"false" + } + + author_id=5 + es_id = create_author(json.loads(json.dumps(test_data)), author_id) + author = Authors.query.filter_by(id=author_id).one() + id = author.json["id"] + + # is_deleted is false, + data={ + "authorNameInfo": [{"familyName": "テスト","firstName": "ハナコ","fullName": "","language": "ja-Kana","nameFormat": "familyNmAndNm","nameShowFlg": "true"}], + "authorIdInfo": [{"idType": "2","authorId": "01234","authorIdShowFlg": "true"}], + "emailInfo": [{"email": "example@com"}], + "is_deleted":False + } + mock_author = MagicMock(spec=Authors) + mock_author.json = { + "id": "551b447c-ad9a-4001-8cb0-60f824579991", + "pk_id": "2", + "emailInfo": [{"email": "example@com"}], + "is_deleted": "false", + "authorIdInfo": [{"idType": "2", "authorId": "01234", "authorIdShowFlg": "true"}], + "authorNameInfo": [ + { + "fullName": "", + "language": "ja-Kana", + "firstName": "ハナコ", + "familyName": "テスト", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true", + } + ], + } + es_author = { + 'hits': { + 'total': 1, + 'hits': [ + { + '_id': id, + '_source': { + } + } + ] + } + } + RecordIndexer().client.search = MagicMock(return_value=es_author) + mock_filter_by =mocker.patch("weko_authors.api.Authors.query.filter_by") + mock_filter_by.return_value.one.return_value = mock_author + WekoAuthors.update(author_id,data) + db.session.commit() + author = Authors.query.filter_by(id=author_id).one() + assert author.json["is_deleted"] == False + res = current_search_client.get(index=current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"],doc_type=current_app.config['WEKO_AUTHORS_ES_DOC_TYPE'],id=es_id) + assert res["_source"]["is_deleted"] == False + + # if es_author['hits']['total'] > 0 is false + test_data = { + "authorNameInfo": [{"familyName": "テスト","firstName": "ハナコ","fullName": "","language": "ja-Kana","nameFormat": "familyNmAndNm","nameShowFlg": "true"}], + "authorIdInfo": [{"idType": "2","authorId": "01234","authorIdShowFlg": "true"}], + "emailInfo": [{"email": "example@com"}], + "is_deleted":"false" + } + + author_id=6 + es_id = create_author(json.loads(json.dumps(test_data)), author_id) + + # is_deleted is false, + data={ + "authorNameInfo": [{"familyName": "テスト","firstName": "ハナコ","fullName": "","language": "ja-Kana","nameFormat": "familyNmAndNm","nameShowFlg": "true"}], + "authorIdInfo": [{"idType": "2","authorId": "01234","authorIdShowFlg": "true"}], + "emailInfo": [{"email": "example@com"}], + "is_deleted":False + } + search_data = { + "hits": { + "total": 0, + "hits": [], + }, + } + mock_indexer = RecordIndexer() + mock_indexer.client = MockClient() + mock_indexer.client.search = MagicMock(return_value=search_data) + mocker.patch("weko_authors.api.RecordIndexer", return_value=mock_indexer) + mocker.patch("weko_authors.api.current_user", return_value=True) + WekoAuthors.update(author_id,data) + db.session.commit() + author = Authors.query.filter_by(id=author_id).one() + assert author.json["is_deleted"] == False + res = current_search_client.get(index=current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"],doc_type=current_app.config['WEKO_AUTHORS_ES_DOC_TYPE'],id=es_id) + assert res["_source"]["is_deleted"] == "false" + # def get_all(cls, with_deleted=True, with_gather=True): # .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_all -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_get_all(self,app,authors): @@ -199,14 +300,151 @@ def test_get_all(self,app,authors): assert authors result = WekoAuthors.get_all(False,False) assert authors + + +# def get_records_count(cls, with_deleted=True, with_gather=True): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_records_count -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_get_records_count(self,app,authors, mocker): + result = WekoAuthors.get_records_count(with_deleted=True, with_gather=True) + assert result == 4 + + result = WekoAuthors.get_records_count(with_deleted=True, with_gather=False) + assert result == 3 + + result = WekoAuthors.get_records_count(with_deleted=False, with_gather=True) + assert result == 3 + + with pytest.raises(Exception): + mocker.patch.object(Authors, 'id', return_value = "test_id") + result = WekoAuthors.get_records_count(with_deleted=False, with_gather=True) + # def get_author_for_validation(cls): # .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_author_for_validation -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_get_author_for_validation(self,authors,mocker): mocker.patch("weko_authors.api.WekoAuthors.get_all",return_value=authors) authors_result, external_result = WekoAuthors.get_author_for_validation() - assert authors_result == {"1":True,"2":True} - assert external_result == {"2":{"1234":["1"],"5678":["2"]}} + assert authors_result == {"1":True,"2":True,"3":True,"4":False} + assert external_result == {"1":{"1":["1"],"2":["2"]},"2":{"1234":["1"],"5678":["2"]},"3":{"12345":["1"]}} + + +# def get_id_prefix_all(cls): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_id_prefix_all -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_get_id_prefix_all(self,authors,authors_prefix_settings,db): + test = [1, 2, 3, 4, 5] + results = WekoAuthors.get_id_prefix_all() + result_list = [] + for result in results: + result_list.append(result.id) + assert result_list == test + + AuthorsPrefixSettings.query.delete() + db.session.commit() + result = WekoAuthors.get_id_prefix_all() + assert result == [] + + + +# def get_scheme_of_id_prefix(cls): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_scheme_of_id_prefix -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_get_scheme_of_id_prefix(self,authors,mocker,authors_prefix_settings,db): + # with patch('weko_authors.api.WekoAuthors.get_scheme_of_id_prefix') as mock_func: + # mock_func.side_effect = WekoAuthors.get_id_prefix_all() + + test = ['WEKO', 'ORCID', 'CiNii', 'KAKEN2', 'ROR'] + results = WekoAuthors.get_scheme_of_id_prefix() + assert results == test + with patch("weko_authors.api.WekoAuthors.get_id_prefix_all", return_value=[]): + results = WekoAuthors.get_scheme_of_id_prefix() + assert results == [] + + + +# def get_affiliation_id_all(cls): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_affiliation_id_all -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_get_affiliation_id_all(self,authors,mocker,authors_affiliation_settings,db): + test = [1, 2, 3, 4] + results = WekoAuthors.get_affiliation_id_all() + result_list = [] + for result in results: + result_list.append(result.id) + assert result_list == test + + AuthorsAffiliationSettings.query.delete() + db.session.commit() + result = WekoAuthors.get_affiliation_id_all() + assert result == [] + + +# def get_scheme_of_affiliaiton_id(cls): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_scheme_of_affiliaiton_id -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_get_scheme_of_affiliaiton_id(self,authors,mocker,authors_affiliation_settings,db): + test = ['ISNI', 'GRID', 'Ringgold', 'kakenhi'] + results = WekoAuthors.get_scheme_of_affiliaiton_id() + assert results == test + with patch("weko_authors.api.WekoAuthors.get_affiliation_id_all", return_value=[]): + results = WekoAuthors.get_scheme_of_affiliaiton_id() + assert results == [] + + +# def test_get_affiliation_identifier_scheme_info(cls): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_affiliation_identifier_scheme_info -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_get_affiliation_identifier_scheme_info(self,authors_affiliation_settings,db): + test = { + "1": {"scheme": "ISNI", "url": "http://www.isni.org/isni/##"}, + "2": {"scheme": "GRID", "url": "https://www.grid.ac/institutes/##"}, + "3": {"scheme": "Ringgold", "url": None}, + "4": {"scheme": "kakenhi", "url": None}, + } + result = WekoAuthors.get_affiliation_identifier_scheme_info() + assert result == test + + AuthorsAffiliationSettings.query.delete() + db.session.commit() + result = WekoAuthors.get_affiliation_identifier_scheme_info() + assert result == {} + + +# def prepare_export_prefix(cls, target_prefix, prefixes): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_prepare_export_prefix -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_prepare_export_prefix(self,authors_affiliation_settings,db,authors_prefix_settings): + data = [ + ['ORCID', 'ORCID', 'https://orcid.org/##'], + ['CiNii', 'CiNii', 'https://ci.nii.ac.jp/author/##'], + ['KAKEN2', 'KAKEN2', 'https://nrid.nii.ac.jp/nrid/##'], + ['ROR', 'ROR', 'https://ror.org/##'] + ] + tests = WekoAuthors.get_id_prefix_all() + result = WekoAuthors.prepare_export_prefix('id_prefix', tests) + assert result == data + + data = [ + ['ISNI', 'ISNI', 'http://www.isni.org/isni/##'], + ['GRID', 'GRID', 'https://www.grid.ac/institutes/##'], + ['Ringgold', 'Ringgold', None], + ['kakenhi', 'kakenhi', None] + ] + tests = WekoAuthors.get_affiliation_id_all() + result = WekoAuthors.prepare_export_prefix('id_prefix', tests) + assert result == data + + data = [ + ['WEKO', 'WEKO', None], + ['ORCID', 'ORCID', 'https://orcid.org/##'], + ['CiNii', 'CiNii', 'https://ci.nii.ac.jp/author/##'], + ['KAKEN2', 'KAKEN2', 'https://nrid.nii.ac.jp/nrid/##'], + ['ROR', 'ROR', 'https://ror.org/##'] + ] + tests = WekoAuthors.get_id_prefix_all() + result = WekoAuthors.prepare_export_prefix('affiliation_id', tests) + assert result == data + + test = [] + result = WekoAuthors.prepare_export_prefix('id_prefix', test) + assert result == [] + + + # def get_identifier_scheme_info(cls): # .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_identifier_scheme_info -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -225,48 +463,745 @@ def test_get_identifier_scheme_info(self,db,authors_prefix_settings): db.session.commit() result = WekoAuthors.get_identifier_scheme_info() assert result == {} + + # def get_by_range(cls, start_point, sum, with_deleted=True, with_gather=True): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_by_range -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_by_range(self, app, authors, mocker): + result = WekoAuthors.get_by_range(0, 10, False, False) + authors_copy = authors + authors_copy.pop() + assert authors_copy == result + result = WekoAuthors.get_by_range(0, 10, True, False) + assert authors == result + result = WekoAuthors.get_by_range(0, 10, False, True) + assert authors == result + with pytest.raises(Exception): + mocker.patch.object(Authors, 'id', return_value = None) + WekoAuthors.get_by_range(0, 10, True, True) + +# def get_pk_id_by_weko_id(cls, weko_id): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_pk_id_by_weko_id -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_pk_id_by_weko_id(self, app, mocker): + data = { + "hits": { + "total": 1, + "hits": [ + { + "_source": { + "authorIdInfo": [ + {"idType": "1", "authorId": "1111", "authorIdShowFlg": "true"}, + {"idType": "2", "authorId": "1111", "authorIdShowFlg": "true"}, + ], + "pk_id": "1", + }, + }, + ], + }, + } + mock_indexer = RecordIndexer() + mocker.patch("weko_authors.api.RecordIndexer",return_value=mock_indexer) + mock_indexer.client = MockClient() + mock_indexer.client.return_value=data + result = WekoAuthors.get_pk_id_by_weko_id("1111") + assert result == "1" + result = WekoAuthors.get_pk_id_by_weko_id("-1") + assert result == -1 -# def prepare_export_data(cls, mappings, authors, schemes): -# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_prepare_export_data -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp - def test_prepare_export_data(self,db, authors,mocker): - mocker.patch("weko_authors.api.WekoAuthors.get_all",return_value=authors) - scheme_info={"1":{"scheme":"WEKO","url":None},"2":{"scheme":"ORCID","url":"https://orcid.org/##"}} - mocker.patch("weko_authors.api.WekoAuthors.get_identifier_scheme_info",return_value=scheme_info) - header, label_en,label_jp,data = WekoAuthors.prepare_export_data(None,None,None) - assert header == ["#pk_id","authorNameInfo[0].familyName","authorNameInfo[0].firstName","authorNameInfo[0].language","authorNameInfo[0].nameFormat","authorNameInfo[0].nameShowFlg","authorIdInfo[0].idType","authorIdInfo[0].authorId","authorIdInfo[0].authorIdShowFlg","emailInfo[0].email","is_deleted"] - assert label_en == ["#WEKO ID","Family Name[0]","Given Name[0]","Language[0]","Name Format[0]","Name Display[0]","Identifier Scheme[0]","Identifier[0]","Identifier Display[0]","Mail Address[0]","Delete Flag"] - assert label_jp == ["#WEKO ID","姓[0]","名[0]","言語[0]","フォーマット[0]","姓名・言語 表示／非表示[0]","外部著者ID 識別子[0]","外部著者ID[0]","外部著者ID 表示／非表示[0]","メールアドレス[0]","削除フラグ"] - - assert data == [["1","テスト","太郎","ja","familyNmAndNm","Y","ORCID","1234","Y","test.taro@test.org",""], - ["2","test","smith","en","familyNmAndNm","Y","ORCID","5678","Y","test.smith@test.org",""]] +# def get_weko_id_by_pk_id(cls, pk_id): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_weko_id_by_pk_id -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_weko_id_by_pk_id(self, app, mocker, authors): + result = WekoAuthors.get_weko_id_by_pk_id("1") + assert result == "1" + result = WekoAuthors.get_weko_id_by_pk_id("-1") + assert result == None + with pytest.raises(Exception): + result = WekoAuthors.get_weko_id_by_pk_id("3") + with pytest.raises(Exception): + result = WekoAuthors.get_weko_id_by_pk_id("4") + with pytest.raises(Exception): + WekoAuthors.get_weko_id_by_pk_id("test_pk_id") - # authors is false - mocker.patch("weko_authors.api.WekoAuthors.get_all",return_value=[]) - header, label_en,label_jp,data = WekoAuthors.prepare_export_data(None,None,None) - assert header == ["#pk_id","authorNameInfo[0].familyName","authorNameInfo[0].firstName","authorNameInfo[0].language","authorNameInfo[0].nameFormat","authorNameInfo[0].nameShowFlg","authorIdInfo[0].idType","authorIdInfo[0].authorId","authorIdInfo[0].authorIdShowFlg","emailInfo[0].email","is_deleted"] - assert label_en == ["#WEKO ID","Family Name[0]","Given Name[0]","Language[0]","Name Format[0]","Name Display[0]","Identifier Scheme[0]","Identifier[0]","Identifier Display[0]","Mail Address[0]","Delete Flag"] - assert label_jp == ["#WEKO ID","姓[0]","名[0]","言語[0]","フォーマット[0]","姓名・言語 表示／非表示[0]","外部著者ID 識別子[0]","外部著者ID[0]","外部著者ID 表示／非表示[0]","メールアドレス[0]","削除フラグ"] + +from sqlalchemy.exc import SQLAlchemyError + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsMappingMaxItem -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestWekoAuthorsMappingMaxItem: +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsMappingMaxItem::test_mapping_max_item_normal_case -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_mapping_max_item_normal_case(self, app, db, authors, mocker): + mappings = [{'json_id': 'authorIdInfo', 'child': [{'json_id': 'idType'}, {'json_id': 'authorId'}]}] + affiliation_mappings = {'json_id': 'affiliationInfo', 'child': [{'json_id': 'identifierInfo', 'child': [{'json_id': 'idType'}, {'json_id': 'affiliationId'}]}]} + records_count = 2 + + result_mappings, result_affiliation_mappings = WekoAuthors.mapping_max_item(mappings, affiliation_mappings, records_count) + assert result_mappings[0]['max'] > 1 + assert 'max' in result_affiliation_mappings - assert data == [] + result_mappings, result_affiliation_mappings = WekoAuthors.mapping_max_item(mappings, None, records_count) + assert result_mappings[0]['max'] > 1 + assert result_affiliation_mappings["max"]== [{'identifierInfo': 2, 'affiliationNameInfo': 1, 'affiliationPeriodInfo': 1}] + # Authors is None + mocker.patch("weko_authors.api.WekoAuthors.get_by_range",return_value=[]) + result_mappings, result_affiliation_mappings = WekoAuthors.mapping_max_item(mappings, None, records_count) + assert result_mappings[0]["max"] == 1 + assert result_affiliation_mappings["max"] == [] + # Author dont has element author = { - "authorNameInfo":[], - "authorIdInfo":[], - "emailInfo":[] + "authorIdInfo": [], + "affiliationInfo": [{ + "affiliationNameInfo": [], + "identifierInfo": [] + }] + } + mappings = [{'json_id': 'authorIdInfo', "max":0, 'child': [{'json_id': 'idType'}, {'json_id': 'authorId'}]}] + authors = [Authors(json=author)] + mocker.patch("weko_authors.api.WekoAuthors.get_by_range",return_value=authors) + result_mappings, result_affiliation_mappings = WekoAuthors.mapping_max_item(mappings, None, records_count) + + assert result_mappings + assert result_affiliation_mappings + + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsMappingMaxItem::test_mapping_max_item_sqlalchemy_error -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_mapping_max_item_sqlalchemy_error(self, app, db, authors): + with patch('weko_authors.api.WekoAuthors.get_records_count', side_effect=SQLAlchemyError): + with pytest.raises(SQLAlchemyError): + WekoAuthors.mapping_max_item(None, None, None) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsMappingMaxItem::test_mapping_max_item_other_exception -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_mapping_max_item_other_exception(self, app, db, authors): + with patch('weko_authors.api.WekoAuthors.get_records_count', side_effect=Exception): + with pytest.raises(Exception): + WekoAuthors.mapping_max_item(None, None, None) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsMappingMaxItem::test_mapping_max_item_retry -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_mapping_max_item_retry(self, app, db, authors, mocker): + mappings = [{'json_id': 'authorIdInfo', 'child': [{'json_id': 'idType'}, {'json_id': 'authorId'}]}] + affiliation_mappings = {'json_id': 'affiliationInfo', 'child': [{'json_id': 'identifierInfo', 'child': [{'json_id': 'idType'}, {'json_id': 'affiliationId'}]}]} + records_count = 2 + + # Mock get_by_range to return a fixed value + mocker.patch("weko_authors.api.WekoAuthors.get_by_range", return_value=authors) + + # Mock db.session.rollback and sleep to verify retry mechanism + with patch('weko_authors.api.db.session.rollback') as mock_rollback, \ + patch('weko_authors.api.sleep') as mock_sleep: + + # First call to mapping_max_item raises SQLAlchemyError, second call succeeds + with patch("weko_authors.api.WekoAuthors.get_records_count", side_effect=[SQLAlchemyError, 2]): + result_mappings, result_affiliation_mappings = WekoAuthors.mapping_max_item(mappings, affiliation_mappings, None) + + # Verify that rollback and sleep were called + mock_rollback.assert_called_once() + mock_sleep.assert_called_once() + + # Verify the results + assert result_mappings + assert result_affiliation_mappings + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsPrepareExport -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestWekoAuthorsPrepareExport: + # .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsPrepareExport::test_prepare_export_data_full_data -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_prepare_export_data_full_data(self, app, db, authors_prefix_settings, authors_affiliation_settings, authors2, mocker): + res_header, res_label_en, res_label_jp, res_row_data = WekoAuthors.prepare_export_data(None, None, None, None, None, 0, 2) + + assert res_header + assert res_label_en + assert res_label_jp + assert res_row_data == [['1', '1', 'テスト', '太郎', 'ja', 'familyNmAndNm', 'Y', 'ORCID', '1234', 'Y', 'CiNii', '12345', 'Y', 'test.taro@test.org', '', None, None, None, None, None, None, '', 'ja', 'Y', None, None, None, None, None, None, None, None, None, None, None, None, None], ['2', '2', 'test', 'smith', 'en', 'familyNmAndNm', 'Y', 'ORCID', '5678', 'Y', None, None, None, 'test.smith@test.org', '', 'ISNI', '1234', 'Y', 'GRID', '12345', 'Y', '', 'ja', 'Y', None, None, 'ISNI', '1234', 'Y', 'GRID', '12345', 'Y', '', 'ja', 'Y', None, None]] + + + @pytest.fixture + def mock_dependencies(self, app, db): + """テストに必要な依存関係をモック化するフィクスチャ""" + with patch('weko_authors.api.WekoAuthors.mapping_max_item') as mock_mapping_max_item, \ + patch('weko_authors.api.WekoAuthors.get_by_range') as mock_get_by_range, \ + patch('weko_authors.api.WekoAuthors.get_identifier_scheme_info') as mock_get_identifier_scheme_info, \ + patch('weko_authors.api.WekoAuthors.get_affiliation_identifier_scheme_info') as mock_get_affiliation_identifier_scheme_info, \ + patch('weko_authors.api.WekoAuthors.get_records_count') as mock_get_records_count: + + # 各モックのデフォルト設定 + mock_mapping_max_item.return_value = ([], {}) + mock_get_by_range.return_value = [] + mock_get_identifier_scheme_info.return_value = {} + mock_get_affiliation_identifier_scheme_info.return_value = {} + mock_get_records_count.return_value = 0 + + yield { + 'mock_mapping_max_item': mock_mapping_max_item, + 'mock_get_by_range': mock_get_by_range, + 'mock_get_identifier_scheme_info': mock_get_identifier_scheme_info, + 'mock_get_affiliation_identifier_scheme_info': mock_get_affiliation_identifier_scheme_info, + 'mock_get_records_count': mock_get_records_count + } + + + def test_prepare_export_data_all_params_provided(self, app, db, mock_dependencies): + """ + 正常系：全てのパラメータが適切に指定されている場合 + """ + # テストデータ設定 + mappings = [ + {'json_id': 'simple_field', 'label_en': 'Simple Field', 'label_jp': 'シンプルフィールド'}, + {'json_id': 'authorIdInfo', 'label_en': 'Author ID', 'label_jp': '著者ID', 'max': 2, 'child': [ + {'json_id': 'idType', 'label_en': 'ID Type', 'label_jp': 'ID種別'}, + {'json_id': 'authorId', 'label_en': 'Author ID', 'label_jp': '著者ID'} + ]}, + {'json_id': 'masked_field', 'label_en': 'Masked Field', 'label_jp': 'マスクフィールド', 'mask': {'1': 'one', '2': 'two'}}, + {'json_id': 'weko_id', 'label_en': 'WEKO ID', 'label_jp': 'WEKO ID'} + ] + + affiliation_mappings = { + 'json_id': 'affiliationInfo', + 'child': [ + {'json_id': 'identifierInfo', 'child': [ + {'json_id': 'affiliationIdType', 'label_en': 'Affiliation ID Type', 'label_jp': '所属ID種別'} + ]}, + {'json_id': 'affiliationNameInfo', 'child': [ + {'json_id': 'affiliationName', 'label_en': 'Affiliation Name', 'label_jp': '所属名'}, + {'json_id': 'language', 'label_en': 'Language', 'label_jp': '言語', 'mask': {'ja': '日本語', 'en': '英語'}} + ]} + ], + 'max': [ + {'identifierInfo': 1, 'affiliationNameInfo': 2, 'affiliationPeriodInfo': 0} + ] } - a = Authors( - gather_flg=0, - is_deleted=False, - json=author + + authors = [ + MagicMock(json={ + 'simple_field': 'simple value', + 'masked_field': '1', + 'authorIdInfo': [ + {'idType': '1', 'authorId': 'weko123'}, + {'idType': '2', 'authorId': 'orcid456'} + ], + 'affiliationInfo': [ + { + 'identifierInfo': [ + {'affiliationIdType': '1', 'affiliationId': 'aff123'} + ], + 'affiliationNameInfo': [ + {'affiliationName': 'University A', 'language': 'en'}, + {'affiliationName': '大学A', 'language': 'ja'} + ] + } + ] + }) + ] + + schemes = { + '1': {'scheme': 'WEKO', 'url': 'http://weko.example.com'}, + '2': {'scheme': 'ORCID', 'url': 'http://orcid.org'} + } + + aff_schemes = { + '1': {'scheme': 'ROR', 'url': 'http://ror.org'} + } + + start = 0 + size = 10 + + # 関数実行 + result = WekoAuthors.prepare_export_data( + mappings, affiliation_mappings, authors, schemes, aff_schemes, start, size ) - mapping = WEKO_AUTHORS_FILE_MAPPING - header, label_en,label_jp,data = WekoAuthors.prepare_export_data(mapping,[a],scheme_info) - assert header == ["#pk_id","authorNameInfo[0].familyName","authorNameInfo[0].firstName","authorNameInfo[0].language","authorNameInfo[0].nameFormat","authorNameInfo[0].nameShowFlg","authorIdInfo[0].idType","authorIdInfo[0].authorId","authorIdInfo[0].authorIdShowFlg","emailInfo[0].email","is_deleted"] - assert label_en == ["#WEKO ID","Family Name[0]","Given Name[0]","Language[0]","Name Format[0]","Name Display[0]","Identifier Scheme[0]","Identifier[0]","Identifier Display[0]","Mail Address[0]","Delete Flag"] - assert label_jp == ["#WEKO ID","姓[0]","名[0]","言語[0]","フォーマット[0]","姓名・言語 表示／非表示[0]","外部著者ID 識別子[0]","外部著者ID[0]","外部著者ID 表示／非表示[0]","メールアドレス[0]","削除フラグ"] + # 結果検証 + row_header, row_label_en, row_label_jp, row_data = result + + # ヘッダーと表示ラベルの検証 + assert row_header[0].startswith('#') + assert row_label_en[0].startswith('#') + assert row_label_jp[0].startswith('#') - assert data == [[None,None,None,None,None,None,None,None,None,None,None]] + # データ行の検証 + assert len(row_data) == 1 # 1件の著者データ + author_row = row_data[0] + # 単純フィールドが正しく処理されていることを確認 + assert 'simple value' in author_row + + # マスク処理が正しく適用されていることを確認 + assert 'one' in author_row + + # WEKO IDが正しく抽出されていることを確認 + assert 'weko123' in author_row + + # IDスキームが正しく変換されていることを確認 + assert 'ORCID' in author_row + + # 所属情報が正しく処理されていることを確認 + assert 'University A' in author_row + assert '英語' in author_row or 'en' in author_row + + # モック関数が呼び出されていないことを確認（すべてのパラメータが提供されているため） + mock_dependencies['mock_mapping_max_item'].assert_not_called() + mock_dependencies['mock_get_by_range'].assert_not_called() + mock_dependencies['mock_get_identifier_scheme_info'].assert_not_called() + mock_dependencies['mock_get_affiliation_identifier_scheme_info'].assert_not_called() + + + def test_prepare_export_data_auto_fetch_params(self, app, db, mock_dependencies): + """ + 正常系：パラメータが一部指定されていない場合の自動補完 + """ + # モックの準備 + test_mappings = [{'json_id': 'test_field', 'label_en': 'Test', 'label_jp': 'テスト'}] + test_aff_mappings = {'json_id': 'affiliationInfo', 'max': [], 'child': []} + test_authors = [MagicMock(json={'test_field': 'test value'})] + mock_dependencies['mock_mapping_max_item'].return_value = (test_mappings, test_aff_mappings) + mock_dependencies['mock_get_by_range'].return_value = test_authors + mock_dependencies['mock_get_identifier_scheme_info'].return_value = {'1': {'scheme': 'TEST'}} + mock_dependencies['mock_get_affiliation_identifier_scheme_info'].return_value = {'1': {'scheme': 'TEST_AFF'}} + + # 関数実行（すべてNoneで渡す） + result = WekoAuthors.prepare_export_data(None, None, None, None, None, 0, 10) + + # 結果検証 + row_header, row_label_en, row_label_jp, row_data = result + + # 各モックが正しく呼び出されたことを確認 + mock_dependencies['mock_mapping_max_item'].assert_called_once() + mock_dependencies['mock_get_by_range'].assert_called_once_with(0, 10, False, False) + mock_dependencies['mock_get_identifier_scheme_info'].assert_called_once() + mock_dependencies['mock_get_affiliation_identifier_scheme_info'].assert_called_once() + + # 返されたデータが期待通りであることを確認 + assert '#test_field' in row_header[0] + assert '#Test' in row_label_en[0] + assert '#テスト' in row_label_jp[0] + assert row_data[0][0] == 'test value' + + + def test_prepare_export_data_different_mapping_types(self, app, db, mock_dependencies): + """ + 正常系：異なる種類のmappingパターンの処理 + """ + # 異なるタイプのマッピングを含むテストデータを設定 + mappings = [ + # childなしマッピング + {'json_id': 'simple_field', 'label_en': 'Simple Field', 'label_jp': 'シンプルフィールド'}, + # childありマッピング + {'json_id': 'complex_field', 'label_en': 'Complex Field', 'label_jp': '複合フィールド', 'max': 2, 'child': [ + {'json_id': 'sub_field1', 'label_en': 'Sub Field 1', 'label_jp': 'サブフィールド1'}, + {'json_id': 'sub_field2', 'label_en': 'Sub Field 2', 'label_jp': 'サブフィールド2'} + ]} + ] + + affiliation_mappings = { + 'json_id': 'affiliationInfo', + 'child': [], + 'max': [] + } + + authors = [ + MagicMock(json={ + 'simple_field': 'simple value', + 'complex_field': [ + {'sub_field1': 'value1_1', 'sub_field2': 'value1_2'}, + {'sub_field1': 'value2_1', 'sub_field2': 'value2_2'} + ], + 'affiliationInfo': [] + }) + ] + + # 関数実行 + result = WekoAuthors.prepare_export_data( + mappings, affiliation_mappings, authors, {}, {}, 0, 10 + ) + + # 結果検証 + row_header, row_label_en, row_label_jp, row_data = result + + # ヘッダーに期待される項目が含まれていることを確認 + assert any('simple_field' in header for header in row_header) + assert any('complex_field[0].sub_field1' in header for header in row_header) + assert any('complex_field[1].sub_field2' in header for header in row_header) + + # ラベルに期待される項目が含まれていることを確認 + assert any('Simple Field' in label for label in row_label_en) + assert any('Sub Field 1[0]' in label for label in row_label_en) + assert any('サブフィールド2[1]' in label for label in row_label_jp) + + # データ行の値が正しいことを確認 + author_row = row_data[0] + assert 'simple value' in author_row + assert 'value1_1' in author_row + assert 'value2_2' in author_row + + + def test_prepare_export_data_author_id_info(self, app, db, mock_dependencies): + """ + 正常系：authorIdInfoの特殊処理 + """ + # authorIdInfoを含むテストデータを設定 + mappings = [ + {'json_id': 'authorIdInfo', 'label_en': 'Author ID', 'label_jp': '著者ID', 'max': 2, 'child': [ + {'json_id': 'idType', 'label_en': 'ID Type', 'label_jp': 'ID種別'}, + {'json_id': 'authorId', 'label_en': 'Author ID', 'label_jp': '著者ID'} + ]} + ] + + affiliation_mappings = { + 'json_id': 'affiliationInfo', + 'child': [], + 'max': [] + } + + authors = [ + MagicMock(json={ + 'authorIdInfo': [ + {'idType': '1', 'authorId': 'weko123'}, + {'idType': '2', 'authorId': 'orcid456'}, + {'idType': '3', 'authorId': 'scopus789'} + ], + 'affiliationInfo': [] + }) + ] + + schemes = { + '1': {'scheme': 'WEKO', 'url': 'http://weko.example.com'}, + '2': {'scheme': 'ORCID', 'url': 'http://orcid.org'}, + '3': {'scheme': 'Scopus', 'url': 'http://scopus.com'} + } + + # 関数実行 + result = WekoAuthors.prepare_export_data( + mappings, affiliation_mappings, authors, schemes, {}, 0, 10 + ) + + # 結果検証 + row_header, row_label_en, row_label_jp, row_data = result + + # IDが正しく処理されていることを確認 + author_row = row_data[0] + + # WEKO ID（index 0）がスキップされ、index 1と2のデータが処理されていることを確認 + assert 'ORCID' in author_row + assert 'orcid456' in author_row + assert 'Scopus' in author_row + assert 'scopus789' in author_row + + + @pytest.fixture + def sample_mappings(self): + """Fixture for sample mappings.""" + return [ + { + 'json_id': 'familyName', + 'label_en': 'Family Name', + 'label_jp': '姓' + }, + { + 'json_id': 'firstName', + 'label_en': 'First Name', + 'label_jp': '名' + }, + { + 'json_id': 'gender', + 'label_en': 'Gender', + 'label_jp': '性別', + 'mask': {'m': 'Male', 'f': 'Female', 'o': 'Other'} + }, + { + 'json_id': 'authorIdInfo', + 'label_en': 'Author ID', + 'label_jp': '著者ID', + 'max': 2, + 'child': [ + { + 'json_id': 'idType', + 'label_en': 'ID Type', + 'label_jp': 'ID種別' + }, + { + 'json_id': 'authorId', + 'label_en': 'Author ID', + 'label_jp': '著者ID' + } + ] + }, + { + 'json_id': 'weko_id', + 'label_en': 'WEKO ID', + 'label_jp': 'WEKO ID' + } + ] + + + @pytest.fixture + def sample_affiliation_mappings(self): + """Fixture for sample affiliation mappings.""" + return { + 'json_id': 'affiliationInfo', + 'child': [ + { + 'json_id': 'identifierInfo', + 'child': [ + { + 'json_id': 'affiliationIdType', + 'label_en': 'Affiliation ID Type', + 'label_jp': '所属ID種別' + }, + { + 'json_id': 'affiliationId', + 'label_en': 'Affiliation ID', + 'label_jp': '所属ID' + } + ] + }, + { + 'json_id': 'affiliationNameInfo', + 'child': [ + { + 'json_id': 'language', + 'label_en': 'Language', + 'label_jp': '言語', + 'mask': {'en': 'English', 'ja': 'Japanese'} + }, + { + 'json_id': 'affiliationName', + 'label_en': 'Affiliation Name', + 'label_jp': '所属名' + } + ] + } + ], + 'max': [ + { + 'identifierInfo': 2, + 'affiliationNameInfo': 2, + 'affiliationPeriodInfo': 1 + } + ] + } + + + @pytest.fixture + def sample_authors(self): + """Fixture for sample authors.""" + return [ + MagicMock( + json={ + 'familyName': 'Smith', + 'firstName': 'John', + 'gender': 'm', + 'authorIdInfo': [ + {'idType': '2', 'authorId': 'ORCID001'} + ], + 'affiliationInfo': [ + { + 'identifierInfo': [ + {'affiliationIdType': '1', 'affiliationId': 'UNI001'}, + {'affiliationIdType': '2', 'affiliationId': 'GRID001'} + ], + 'affiliationNameInfo': [ + {'language': 'en', 'affiliationName': 'University A'}, + {'language': 'ja', 'affiliationName': '大学A'} + ] + } + ] + } + ) + ] + + + @pytest.fixture + def sample_schemes(self): + """Fixture for sample ID schemes.""" + return { + '1': {'scheme': 'WEKO', 'url': 'https://weko.org'}, + '2': {'scheme': 'ORCID', 'url': 'https://orcid.org'} + } + + + @pytest.fixture + def sample_aff_schemes(self): + """Fixture for sample affiliation ID schemes.""" + return { + '1': {'scheme': 'University ID', 'url': 'https://university.org'}, + '2': {'scheme': 'GRID', 'url': 'https://grid.ac'} + } + + + def test_mask_processing(self, app, db, sample_mappings, sample_affiliation_mappings, sample_authors, + sample_schemes, sample_aff_schemes): + """Test case 7: mask processing is correctly applied.""" + row_header, row_label_en, row_label_jp, row_data = WekoAuthors.prepare_export_data( + sample_mappings, + sample_affiliation_mappings, + sample_authors, + sample_schemes, + sample_aff_schemes, + 0, + 10 + ) + + # Check if mask processing was applied for gender + assert row_data[0][2] == 'Male' # The gender 'm' should be masked to 'Male' + + # Check if mask processing was applied for affiliation language + # Finding the index for affiliationInfo[0].affiliationNameInfo[0].language + language_index = row_header.index('affiliationInfo[0].affiliationNameInfo[0].language') + assert row_data[0][language_index] == 'English' # 'en' should be masked to 'English' + + + def test_affiliation_info_processing(self, app, db, sample_mappings, sample_affiliation_mappings, sample_authors, + sample_schemes, sample_aff_schemes): + """Test case 8: affiliation information is correctly processed.""" + row_header, row_label_en, row_label_jp, row_data = WekoAuthors.prepare_export_data( + sample_mappings, + sample_affiliation_mappings, + sample_authors, + sample_schemes, + sample_aff_schemes, + 0, + 10 + ) + + # Check if affiliationIdType is correctly processed with scheme lookup + # Find index for affiliationInfo[0].identifierInfo[0].affiliationIdType + aff_id_type_index = row_header.index('affiliationInfo[0].identifierInfo[0].affiliationIdType') + assert row_data[0][aff_id_type_index] == 'University ID' + + # Check if affiliationId is correctly included + aff_id_index = row_header.index('affiliationInfo[0].identifierInfo[0].affiliationId') + assert row_data[0][aff_id_index] == 'UNI001' + + # Check if affiliationName is correctly included + aff_name_index = row_header.index('affiliationInfo[0].affiliationNameInfo[0].affiliationName') + assert row_data[0][aff_name_index] == 'University A' + + # Check if headers and labels are correctly formatted for nested structure + assert 'affiliationInfo[0].identifierInfo[0].affiliationIdType' in row_header + assert 'Affiliation ID Type[0][0]' in row_label_en + assert '所属ID種別[0][0]' in row_label_jp + + + @patch('weko_authors.api.WekoAuthors.mapping_max_item') + @patch('weko_authors.api.WekoAuthors.get_by_range') + @patch('weko_authors.api.WekoAuthors.get_identifier_scheme_info') + @patch('weko_authors.api.WekoAuthors.get_affiliation_identifier_scheme_info') + def test_affiliation_info_missing_data(self, mock_aff_scheme, mock_scheme, mock_get, mock_map, + sample_mappings, sample_affiliation_mappings, + sample_schemes, sample_aff_schemes, app, db): + """Test case 8: handling of missing affiliation data.""" + # Author with partial affiliation data + authors_with_missing = [ + MagicMock( + json={ + 'familyName': 'Doe', + 'firstName': 'Jane', + 'gender': 'f', + 'authorIdInfo': [ + {'idType': '1', 'authorId': 'WEKO002'} + ], + 'affiliationInfo': [ + { + # Missing identifierInfo + 'affiliationNameInfo': [ + {'language': 'en', 'affiliationName': 'University B'} + # Only one name, not two as in max + ] + } + ] + } + ) + ] + + # Configure mocks to return our test data + mock_map.return_value = (sample_mappings, sample_affiliation_mappings) + mock_get.return_value = authors_with_missing + mock_scheme.return_value = sample_schemes + mock_aff_scheme.return_value = sample_aff_schemes + + row_header, row_label_en, row_label_jp, row_data = WekoAuthors.prepare_export_data( + None, # Let the method fetch mappings + None, # Let the method fetch affiliation mappings + None, # Let the method fetch authors + None, # Let the method fetch schemes + None, # Let the method fetch affiliation schemes + 0, + 10 + ) + + # Check if missing affiliationIdType is correctly handled as None + aff_id_type_index = row_header.index('affiliationInfo[0].identifierInfo[0].affiliationIdType') + assert row_data[0][aff_id_type_index] is None + + # Check if missing second affiliationNameInfo is correctly handled as None + aff_name_index = row_header.index('affiliationInfo[0].affiliationNameInfo[1].affiliationName') + assert row_data[0][aff_name_index] is None + + # Verify all expected headers are present + assert 'affiliationInfo[0].identifierInfo[1].affiliationId' in row_header + assert 'affiliationInfo[0].affiliationNameInfo[0].language' in row_header + + # Verify headers and labels match + assert len(row_header) == len(row_label_en) + assert len(row_header) == len(row_label_jp) + +# def get_used_scheme_of_id_prefix(cls): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsGetUsedSchemeOfIdPrefix -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestWekoAuthorsGetUsedSchemeOfIdPrefix: +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_used_scheme_of_id_prefix_1 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_used_scheme_of_id_prefix_1(self, app, authors, authors_prefix_settings): + result = WekoAuthors.get_used_scheme_of_id_prefix() + assert result == (['ORCID', 'CiNii'], {1: 'WEKO', 2: 'ORCID', 3: 'CiNii', 4: 'KAKEN2', 5: 'ROR'}) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_used_scheme_of_id_prefix_2 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_used_scheme_of_id_prefix_2(self, app, authors_prefix_settings): + result = WekoAuthors.get_used_scheme_of_id_prefix() + assert result == ([], {1: 'WEKO', 2: 'ORCID', 3: 'CiNii', 4: 'KAKEN2', 5: 'ROR'}) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_used_scheme_of_id_prefix_3 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_used_scheme_of_id_prefix_3(self, app, authors): + result = WekoAuthors.get_used_scheme_of_id_prefix() + assert result == ([None], {}) + +# def get_used_scheme_of_affiliation_id(cls): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthorsGetUsedSchemeOfAffiliationId -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestWekoAuthorsGetUsedSchemeOfAffiliationId: +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_used_scheme_of_affiliation_id_1 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_used_scheme_of_affiliation_id_1(self, app, authors, authors_affiliation_settings): + result = WekoAuthors.get_used_scheme_of_affiliation_id() + assert result == (['ISNI', 'GRID'], {1: 'ISNI', 2: 'GRID', 3: 'Ringgold', 4: 'kakenhi'}) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_used_scheme_of_affiliation_id_2 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_used_scheme_of_affiliation_id_2(self, app, authors_affiliation_settings): + result = WekoAuthors.get_used_scheme_of_affiliation_id() + assert result == ([], {1: 'ISNI', 2: 'GRID', 3: 'Ringgold', 4: 'kakenhi'}) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_api.py::TestWekoAuthors::test_get_used_scheme_of_affiliation_id_3 -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_get_used_scheme_of_affiliation_id_3(self, app, authors): + result = WekoAuthors.get_used_scheme_of_affiliation_id() + assert result == ([None], {}) \ No newline at end of file diff --git a/modules/weko-authors/tests/test_rest.py b/modules/weko-authors/tests/test_rest.py new file mode 100644 index 0000000000..f6da091529 --- /dev/null +++ b/modules/weko-authors/tests/test_rest.py @@ -0,0 +1,485 @@ +# .tox/c1/bin/pytest --cov=weko_authors tests/test_rest.py -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp + +import json +from mock import patch +from flask import Blueprint, Response + +from invenio_deposit.utils import check_oauth2_scope_write, \ + check_oauth2_scope_write_elasticsearch +from invenio_records_rest.utils import check_elasticsearch +from weko_authors.rest import ( + create_blueprint, +) + +import urllib.parse +from flask_oauthlib.provider import OAuth2Provider +from invenio_oauth2server.views.server import login_oauth2_user +from mock import MagicMock +from weko_index_tree.api import Indexes +import pytest +from sqlalchemy.exc import SQLAlchemyError + +blueprint = Blueprint( + 'invenio_records_rest', + __name__, + url_prefix='', +) + +_PID = 'pid(depid,record_class="invenio_deposit.api:Deposit")' + +endpoints = { + 'depid': { + 'pid_type': 'depid', + 'pid_minter': 'deposit', + 'pid_fetcher': 'deposit', + 'record_class': 'invenio_deposit.api:Deposit', + 'files_serializers': { + 'application/json': ('invenio_deposit.serializers:json_v1_files_response'), + }, + 'search_class': 'invenio_deposit.search:DepositSearch', + 'search_serializers': { + 'application/json': ('invenio_records_rest.serializers:json_v1_search'), + }, + 'list_route': '/deposits/', + 'indexer_class': None, + 'item_route': '/deposits/<{0}:pid_value>'.format(_PID), + 'route': '/deposits/<{0}:pid_value>/authors/count'.format(_PID), + 'default_media_type': 'application/json', + 'links_factory_imp': 'invenio_deposit.links:deposit_links_factory', + 'create_permission_factory_imp': check_oauth2_scope_write, + 'read_permission_factory_imp': check_elasticsearch, + 'update_permission_factory_imp': + check_oauth2_scope_write_elasticsearch, + 'delete_permission_factory_imp': + check_oauth2_scope_write_elasticsearch, + 'max_result_window': 10000, + 'cites_route': 1, + }, +} + + +# def create_blueprint(endpoints): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_rest.py::test_create_blueprint -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_create_blueprint(app): + assert create_blueprint(endpoints) != None + + +# def Authors(ContentNegotiatedMethodView): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_rest.py::test_Authors -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_Authors(app): + app.register_blueprint(create_blueprint(app.config['WEKO_AUTHORS_REST_ENDPOINTS'])) + with \ + app.test_client() as client, \ + patch('weko_authors.utils.count_authors', return_value=Response(status=200)), \ + patch('json.dumps', return_value={}): + + # 1 GET reqest + res = client.get('/v1/authors/count') + assert res.status_code == 200 + + # 2 Invalid version + res = client.get('/v0/authors/count') + assert res.status_code == 400 + + + + +# .tox/c1/bin/pytest --cov=weko_index_tree tests/test_rest.py::TestAuthorDBManagementAPI -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko_index_tree/.tox/c1/tmp --full-trace -p no:warnings + +class TestAuthorDBManagementAPI: + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + # .tox/c1/bin/pytest --cov=weko_authors tests/test_rest.py::TestAuthorDBManagementAPI::test_get_v1 -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko_index_tree/.tox/c1/tmp --full-trace | tee log.log + def test_get_v1(self, app, esindex, client_api, auth_headers_noroleuser, auth_headers_sysadmin, auth_headers_sysadmin_without_scope, author_records_for_test, authors_affiliation_settings, authors_prefix_settings): + """ + 著者DB検索API - 著者情報取得 + - 正常系: 検索パラメータごとに正しくデータが取得できるか確認 + - 異常系: 不正なパラメータや認証なしでのアクセス制御確認 + """ + + oauth2 = OAuth2Provider() + oauth2.after_request(login_oauth2_user) + + # 正常系テスト（検索条件ごとの動作確認） + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"fullname": "Test_1 User_1"}, 200, ['1']) # フルネーム検索で特定の著者ID 1 が返ることを確認 + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"fullname": "Test_1 "}, 200, ['1','2','3','4']) # 部分一致検索で該当する複数の著者IDが返ることを確認 + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"firstname": "Test_2"}, 200, ['2']) # 名（ファーストネーム）で検索し、著者ID 2 のみが返ることを確認 + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"familyname": "User_3"}, 200, ['3','4']) # 姓（ファミリーネーム）で検索し、著者ID 3 のみが返ることを確認 + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"idtype": "WEKO", "authorid": "1"}, 200, ['1']) # ID タイプが WEKO で ID が 1 の著者が正しく取得できるか確認 + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"idtype": "ORCID", "authorid": "1"}, 200, []) # ID タイプが WEKO で ID が 1 の著者が正しく取得できるか確認 + + # 異常系テスト（不正なアクセスや無効なパラメータの確認） + self.run_get_authors_unauthorized(app, client_api) # 認証なしでのアクセスが 401（Unauthorized） となることを確認 + self.run_get_authors(app, client_api, auth_headers_sysadmin_without_scope, {"fullname": "Test_1 "}, 403, []) # スコープ権限のないシステム管理者が検索した場合、403（Forbidden）となることを確認 + self.run_get_authors(app, client_api, auth_headers_noroleuser, {"fullname": "Test_1 "}, 403, []) # 権限のないユーザーが検索した場合、403（Forbidden）となることを確認 + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"idtype": "ORCID"}, 400, []) # Both 'idtype' and 'authorid' must be specified together or omitted. + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"idtype": "ORCID_wrong","authorid": "1"}, 400, []) # ORCID というIDタイプでの検索が不正な場合、400（Bad Request）となることを確認 + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"authorid": "https://orcid.org/##"}, 400, []) # 無効な著者IDで検索した場合、400（Bad Request）となることを確認 + + with patch("weko_authors.utils.get_author_prefix_obj",side_effect={}): + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"fullname": "Test_1 User_1"}, 200, ['1']) # フルネーム検索で特定の著者ID 1 が返ることを確認 + # システムエラーの確認 + with patch("invenio_search.current_search_client.search", side_effect=Exception): + self.run_get_authors(app, client_api, auth_headers_sysadmin, {"firstname": "Test_2"}, 500, []) # 検索時にエラーが発生した場合、500（Internal Server Error）となることを確認 + + def run_get_authors(self, app, client_api, user_role, query_params, expected_status, expected_ids): + """ + 著者情報取得APIのテスト + - 検索条件に応じて正しくデータが取得できるか確認 + - 取得した著者IDリストが期待通りか検証 + """ + if query_params: + url = f"v1/authors?{self.build_query_string(query_params)}" + else: + url = f"v1/authors" + response = client_api.get(url, headers=user_role) + assert response.status_code == expected_status + if response.status_code == 200: + data = response.json + assert "authors" in data, "レスポンスJSONに 'authors' キーが含まれていない" + print(data) + retrieved_ids = [ + authorInfo.get("authorId", None) + for author in data.get("authors", []) + for authorInfo in author.get("authorIdInfo", []) + if authorInfo.get("idType") == "WEKO" + ] + assert set(retrieved_ids) == set(expected_ids), f"取得した著者IDが想定外: {retrieved_ids}" + + def run_get_authors_unauthorized(self, app, client_api): + """ + 認証なしで著者情報取得を試みるテスト + - 認証が必要なAPIにアクセスし、Unauthorized(401)が返ることを確認 + """ + url = "v1/authors" + response = client_api.get(url, headers={}) + assert response.status_code == 401 + print(f"Unauthorizedアクセスエラー: {response.get_data(as_text=True)}") + + def build_query_string(self, params): + """ + クエリパラメータをURLエンコードするヘルパー関数 + """ + return "&".join(f"{key}={urllib.parse.quote(str(value))}" for key, value in params.items() if value) + + # .tox/c1/bin/pytest --cov=weko_authors tests/test_rest.py::TestAuthorDBManagementAPI::test_post_v1 -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko_index_tree/.tox/c1/tmp --full-trace | tee log.log + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + def test_post_v1(self, app, client_api, auth_headers_noroleuser, auth_headers_sysadmin, author_records_for_test, authors_affiliation_settings, authors_prefix_settings): + """ + 著者情報登録API - 著者情報の登録テスト + - 正常系: 正しく登録できるか確認 + - 異常系: 不正なリクエストや認証なしでのアクセス制御確認 + """ + + oauth2 = OAuth2Provider() + oauth2.after_request(login_oauth2_user) + + # 正常系テスト + # 正しく著者情報を登録できることを確認 + self.run_post_author(app, client_api, auth_headers_sysadmin, self.valid_author_data(), 200, "Author successfully registered.") # 正常なデータ + self.run_post_author(app, client_api, auth_headers_sysadmin, self.valid_author_data("ORCID", ""), 200, "Author successfully registered.") # カーバレジのためのテストケース + self.run_post_author(app, client_api, auth_headers_sysadmin, self.valid_author_data("ORCID", ""), 200, "Author successfully registered.") # カーバレジのためのテストケース + + # 認証なしのリクエストが拒否されることを確認 + self.run_post_author_unauthorized(app, client_api) # 認証なしでリクエストするとエラー + + # 権限のないユーザーのリクエストが拒否されることを確認 + self.run_post_author(app, client_api, auth_headers_noroleuser, self.valid_author_data(), 403, None) # 権限なしのユーザーでリクエスト + + # 異常系テスト（リクエスト内容のバリデーション） + # 不正なリクエストが適切に処理されることを確認 + self.run_post_author(app, client_api, auth_headers_sysadmin, {}, 400, "Request body cannot be empty.") # 空のリクエスト + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": None}, 400, "author can not be null.") # authorがNone + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"authorIdInfo": [{"idType": "ORCID"}]}}, 400, "Both 'idType' and 'authorId' must be provided together.") # idTypeのみ指定 + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"authorIdInfo": [{"idType": "WEKO", "authorId": "A1"}]}}, 400, "The WEKO ID must be numeric characters only.") # WEKO IDが数字以外 + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"authorIdInfo": [{"idType": "WEKO", "authorId": "1"}]}}, 400, "The value is already in use as WEKO ID.") # 既存のWEKO ID + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"authorNameInfo": [{"firstName": "John", "familyName": "Doe"}]}}, 400, "If 'firstName' or 'familyName' is provided, 'language' must also be specified.") # language未指定 + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"affiliationInfo": "InvalidFormat"}}, 400, "'affiliationInfo' should be a list.") # affiliationInfoがリストでない + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"affiliationInfo": ["InvalidFormat"]}}, 400, "Invalid format in 'affiliationInfo'.") # affiliationInfoのフォーマット不正 + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"affiliationInfo": [{"affiliationPeriodInfo": [{"periodStart": "2025-03-21", "periodEnd": "2025-01-27"}]}]}}, 400, "affiliationPeriodInfo error: start is after end") # 開始日が終了日より後 + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"affiliationInfo": [{"identifierInfo": [{"affiliationId": "https://ror.org/##", "identifierShowFlg": "true"}]}]}}, 400, "Both 'affiliationIdType' and 'affiliationId' must be provided together.") # affiliationIdType未指定 + self.run_post_author(app, client_api, auth_headers_sysadmin, {"author": {"affiliationInfo": [{"affiliationNameInfo": [{"affiliationName": "NII", "identifierShowFlg": "true"}]}]}}, 400, "Both 'affiliationName' and 'affiliationNameLang' must be provided together.") # affiliationNameLang未指定 + + # システムエラーの確認 + # DBエラーや例外発生時の動作を確認 + self.run_post_author_db_error(app, client_api, auth_headers_sysadmin, self.valid_author_data()) # DBエラー発生時 + with patch("invenio_search.current_search_client.search", side_effect=Exception): + self.run_post_author(app, client_api, auth_headers_sysadmin, self.valid_author_data(), 500) # 検索時に例外が発生した場合 + + def run_post_author(self, app, client_api, user_headers, request_data, expected_status, expected_message=None): + """ + 著者情報登録APIのテスト + - 指定したリクエストデータでAPIを実行し、期待するレスポンスを検証 + """ + url = "v1/authors" + response = client_api.post(url, json=request_data, headers=user_headers) + assert response.status_code == expected_status, f"Expected {expected_status}, got {response.status_code}" + + if expected_message: + content_type = response.headers.get("Content-Type", "").lower() + if "text/html" in content_type: + # print("Response is HTML format.") + response_text = response.get_data(as_text=True) + assert expected_message in response_text, f"Expected message '{expected_message}' not found in HTML response, got: \n{response_text}" + else: + response_json = json.loads(response.data.decode('utf-8')) + message = response_json.get("message") + assert expected_message in message, f"Expected message: {expected_message}, got: {message}" + + def run_post_author_unauthorized(self, app, client_api): + """ + 認証なしで著者情報を登録しようとした場合のテスト + - 認証が必要なAPIにアクセスし、Unauthorized(401)が返ることを確認 + """ + url = "v1/authors" + response = client_api.post(url, json=self.valid_author_data(), headers={}) + + assert response.status_code == 401 + + def run_post_author_db_error(self, app, client_api, user_headers, request_data): + """ + データベースエラー時のテスト + - DBエラー発生時に500 (Internal Server Error) が返されることを確認 + """ + url = "v1/authors" + + with patch("weko_authors.models.Authors.get_sequence", side_effect=SQLAlchemyError): + response = client_api.post(url, json=request_data, headers=user_headers) + + assert response.status_code == 500 + assert "Database error." in response.get_data(as_text=True) + + def valid_author_data(self, idType = "WEKO",nameFormat = "familyNmAndNm"): + """ + 正常な著者データ + - 正しく登録できるデータを返す + """ + return { + "author": { + "emailInfo": [{"email": "sample@xxx.co.jp"}], + "authorIdInfo": [{"idType": idType, "authorId": "5", "authorIdShowFlg": True}], + "authorNameInfo": [{"language": "en", "firstName": "John", "familyName": "Doe", "nameFormat": nameFormat, "nameShowFlg": True}], + "affiliationInfo": [{ + "identifierInfo": [{"affiliationId": "https://ror.org/5678", "affiliationIdType": "ISNI", "identifierShowFlg": True}], + "affiliationNameInfo": [{"affiliationName": "NII", "affiliationNameLang": "en", "affiliationNameShowFlg": True}], + "affiliationPeriodInfo": [{"periodStart": "2025-01-27", "periodEnd": "2025-03-21"}] + }] + } + } + + + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + # .tox/c1/bin/pytest --cov=weko_authors tests/test_rest.py::TestAuthorDBManagementAPI::test_put_v1 -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko_index_tree/.tox/c1/tmp --full-trace | tee log.log + def test_put_v1(self, app, client_api, auth_headers_noroleuser, auth_headers_sysadmin, author_records_for_test, authors_affiliation_settings, authors_prefix_settings): + """ + 著者情報更新API - 著者情報の更新テスト + - 正常系: 正しく更新できるか確認 + - 異常系: 不正なリクエストや認証なしでのアクセス制御確認 + """ + with patch("weko_deposit.tasks.update_items_by_authorInfo.delay",side_effect = MagicMock()): + oauth2 = OAuth2Provider() + oauth2.after_request(login_oauth2_user) + + # 正常系テスト + # 正しく著者情報を更新できることを確認 + self.run_put_author(app, client_api, auth_headers_sysadmin, self.valid_update_data(), 200, "Author successfully updated.") + author_data = self.valid_update_data() + del author_data["author"]["emailInfo"] + self.run_put_author(app, client_api, auth_headers_sysadmin, author_data, 200, "Author successfully updated.") + es_id = author_records_for_test["1"] + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": es_id, "author": self.valid_update_data().get("author")}, 200, "Author successfully updated.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": es_id, "force_change": True, "author": self.valid_update_data().get("author")}, 200, "Author successfully updated.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": es_id, "pk_id": "1", "author": self.valid_update_data().get("author")}, 200, "Author successfully updated.") + + data_no_weko = { + "pk_id": "1", + "author": { + "emailInfo": [{"email": "updated@xxx.co.jp"}], + "authorIdInfo": [{"idType": "ORCID", "authorId": "5", "authorIdShowFlg": True}], + "authorNameInfo": [{"language": "en", "firstName": "Jane", "familyName": "Smith", "nameFormat": "familyNmAndNm", "nameShowFlg": True}], + "affiliationInfo": [{ + "identifierInfo": [{"affiliationId": "https://ror.org/5678", "affiliationIdType": "ISNI", "identifierShowFlg": True}], + "affiliationNameInfo": [{"affiliationName": "NII", "affiliationNameLang": "en", "affiliationNameShowFlg": True}], + "affiliationPeriodInfo": [{"periodStart": "2025-02-01", "periodEnd": "2025-04-01"}] + }] + } + } + self.run_put_author(app, client_api, auth_headers_sysadmin, data_no_weko, 400, "idType: WEKO (weko id) must be provided.") + + # 認証なしのリクエストが拒否されることを確認 + self.run_put_author_unauthorized(app, client_api) + + # 権限のないユーザーのリクエストが拒否されることを確認 + self.run_put_author(app, client_api, auth_headers_noroleuser, self.valid_update_data(), 403, None) + + # 異常系テスト（リクエスト内容のバリデーション） + self.run_put_author(app, client_api, auth_headers_sysadmin, {}, 400, "Request body cannot be empty.") + author_data = self.valid_update_data() + author_data["author"]["authorNameInfo"][0]["language"] = "jp" + self.run_put_author(app, client_api, auth_headers_sysadmin, author_data, 400, "Invalid Author Data, 'idtype' or 'language' Not Allowed.") + author_data = self.valid_update_data() + author_data["author"]["affiliationInfo"][0]["affiliationNameInfo"][0]["affiliationNameLang"] = "jp" + self.run_put_author(app, client_api, auth_headers_sysadmin, author_data, 400, "Invalid Author Data, 'idtype' or 'language' Not Allowed.") + author_data = self.valid_update_data() + author_data["author"]["authorIdInfo"][0]["idType"] = "WEKO_WRONG" + self.run_put_author(app, client_api, auth_headers_sysadmin, author_data, 400, "Invalid Author Data, 'idtype' or 'language' Not Allowed.") + author_data = self.valid_update_data() + author_data["author"]["affiliationInfo"][0]["identifierInfo"][0]["affiliationIdType"] = "Ringgold_WRONG" + self.run_put_author(app, client_api, auth_headers_sysadmin, author_data, 400, "Invalid Author Data, 'idtype' or 'language' Not Allowed.") + author_data = self.valid_update_data() + author_data.pop("pk_id") + self.run_put_author(app, client_api, auth_headers_sysadmin, author_data, 400, "Bad Request: Either 'id' or 'pk_id' must be specified.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": "123"}, 400, "author can not be null.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"pk_id": "123"}, 400, "author can not be null.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": "invalid", "pk_id": "invalid", "author": self.valid_update_data().get("author")}, 400, "Invalid author ID.") + + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": "999", "pk_id": "999", "author": self.valid_update_data().get("author")}, 404, "Specified author does not exist.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"pk_id": "999", "author": self.valid_update_data().get("author")}, 404, "Specified author does not exist.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": "999", "author": self.valid_update_data().get("author")}, 404, "Specified author does not exist.") + + # IDが異なるケース + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": "123", "pk_id": "1", "author": self.valid_update_data()["author"]}, 400, "Parameters 'id' and 'pk_id' refer to different users.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": "123", "pk_id": "1", "author": self.valid_update_data()["author"]}, 400, "Parameters 'id' and 'pk_id' refer to different users.") + self.run_put_author(app, client_api, auth_headers_sysadmin, {"id": es_id, "pk_id": "2", "author": self.valid_update_data().get("author")}, 400, "Parameters 'id' and 'pk_id' refer to different users.") + + # システムエラーの確認 + self.run_put_author_db_error(app, client_api, auth_headers_sysadmin, self.valid_update_data()) + + def run_put_author(self, app, client_api, user_headers, request_data, expected_status, expected_message=None): + """ + 著者情報更新APIのテスト + """ + print(1111111111111111111111111111) + print(request_data) + print(1111111111111111111111111111) + url = "v1/authors" + response = client_api.put(url, json=request_data, headers=user_headers) + # assert response.status_code == expected_status, f"Expected {expected_status}, got {response.status_code}" + + if expected_message: + content_type = response.headers.get("Content-Type", "").lower() + if "text/html" in content_type: + response_text = response.get_data(as_text=True) + assert expected_message in response_text, f"Expected message '{expected_message}' not found in HTML response, got: \n{response_text}" + else: + response_json = json.loads(response.data.decode('utf-8')) + message = response_json.get("message") + assert expected_message in message, f"Expected message: {expected_message}, got: {message}" + + def run_put_author_unauthorized(self, app, client_api): + """ + 認証なしで著者情報を更新しようとした場合のテスト + """ + url = "v1/authors" + response = client_api.put(url, json=self.valid_update_data(), headers={}) + assert response.status_code == 401 + + def run_put_author_db_error(self, app, client_api, user_headers, request_data): + """ + データベースエラー時のテスト + """ + url = "v1/authors" + with patch("weko_authors.models.Authors.query", side_effect=SQLAlchemyError): + response = client_api.put(url, json=request_data, headers=user_headers) + assert response.status_code == 500 + assert "Database error." in response.get_data(as_text=True) + + with patch("weko_authors.rest.AuthorDBManagementAPI.get_all_schemes", side_effect=Exception): + response = client_api.put(url, json=request_data, headers=user_headers) + assert response.status_code == 500 + assert "Internal server error." in response.get_data(as_text=True) + + def valid_update_data(self): + """ + 正常な更新データ + """ + return { + "pk_id": "1", + "author": { + "emailInfo": [{"email": "updated@xxx.co.jp"}], + "authorIdInfo": [{"idType": "WEKO", "authorId": "5", "authorIdShowFlg": True}], + "authorNameInfo": [{"language": "en", "firstName": "Jane", "familyName": "Smith", "nameFormat": "familyNmAndNm", "nameShowFlg": True}], + "affiliationInfo": [{ + "identifierInfo": [{"affiliationId": "https://ror.org/5678", "affiliationIdType": "ISNI", "identifierShowFlg": True}], + "affiliationNameInfo": [{"affiliationName": "NII", "affiliationNameLang": "en", "affiliationNameShowFlg": True}], + "affiliationPeriodInfo": [{"periodStart": "2025-02-01", "periodEnd": "2025-04-01"}] + }] + } + } + + @pytest.mark.parametrize('base_app',[dict( + is_es=True + )],indirect=['base_app']) + # .tox/c1/bin/pytest --cov=weko_authors tests/test_rest.py::TestAuthorDBManagementAPI::test_delete_v1 -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko_index_tree/.tox/c1/tmp --full-trace | tee log.log + def test_delete_v1(self, app, client_api, auth_headers_sysadmin, auth_headers_noroleuser,author_records_for_test): + """ + 著者削除API - 著者情報の削除テスト + - 正常系: 正しく削除できるか確認 + - 異常系: 認証なし、権限なし、存在しないデータ削除時の動作確認 + """ + es_ids = author_records_for_test + self.run_delete_author(app, client_api, auth_headers_sysadmin, {"pk_id": "1"}, 200) + self.run_delete_author(app, client_api, auth_headers_sysadmin, {"id": es_ids["2"]}, 200) + self.run_delete_author(app, client_api, auth_headers_sysadmin, {"pk_id": "1", "id": es_ids["3"]}, 400, "Parameters 'id' and 'pk_id' refer to different users.") + self.run_delete_author(app, client_api, auth_headers_sysadmin, {"pk_id": "3", "id": es_ids["3"]}, 200) + + self.run_delete_author(app, client_api, auth_headers_sysadmin, {}, 400, "Request body cannot be empty.") + self.run_delete_author(app, client_api, auth_headers_sysadmin, {"abc":"abc"}, 400, "Either 'id' or 'pk_id' must be specified.") + self.run_delete_author(app, client_api, auth_headers_sysadmin, {"pk_id": "999"}, 404, "Specified author does not exist.") + self.run_delete_author(app, client_api, auth_headers_sysadmin, {"id": "es_not_found"}, 404, "Specified author does not exist.") + + # 認証エラー + # self.run_delete_author_unauthorized(app, client_api) + + # 権限なし + self.run_delete_author(app, client_api, auth_headers_noroleuser, {"pk_id": "1"}, 403) + + # エラー + self.run_delete_author_error(app, client_api, auth_headers_sysadmin, {"pk_id": "4"}) + + def run_delete_author(self, app, client_api, user_headers, request_data, expected_status, expected_message=None): + """ + 著者削除APIのテスト + - 指定したリクエストデータでAPIを実行し、期待するレスポンスを検証 + """ + url = "v1/authors" + response = client_api.delete(url, json=request_data, headers=user_headers) + print(response.get_data(as_text=True)) + assert response.status_code == expected_status, f"Expected {expected_status}, got {response.status_code}" + if expected_message: + assert expected_message in response.get_data(as_text=True) + + def run_delete_author_unauthorized(self, app, client_api): + """ + 認証なしで著者情報を削除しようとした場合のテスト + - 認証が必要なAPIにアクセスし、Unauthorized(401)が返ることを確認 + """ + url = "v1/authors" + response = client_api.delete(url, json={"pk_id": "1"}, headers={}) + + assert response.status_code == 401 + assert "OAuth2 authentication failed" in response.get_data(as_text=True) + + def run_delete_author_error(self, app, client_api, user_headers, request_data): + """ + データベースエラー時のテスト + - DBエラー発生時に500 (Internal Server Error) が返されることを確認 + """ + url = "v1/authors" + + with patch("weko_authors.models.db.session.commit", side_effect=SQLAlchemyError): + response = client_api.delete(url, json=request_data, headers=user_headers) + + assert response.status_code == 500 + assert "Database error." in response.get_data(as_text=True) + + with patch("weko_authors.models.db.session.commit", side_effect=Exception): + response = client_api.delete(url, json=request_data, headers=user_headers) + + assert response.status_code == 500 + assert "Internal server error." in response.get_data(as_text=True) \ No newline at end of file diff --git a/modules/weko-authors/tests/test_tasks.py b/modules/weko-authors/tests/test_tasks.py index 73a3ea3a0f..62566bfddf 100644 --- a/modules/weko-authors/tests/test_tasks.py +++ b/modules/weko-authors/tests/test_tasks.py @@ -1,9 +1,24 @@ -from mock import patch +import json +import pytest +from mock import patch, MagicMock, call, mock_open, call +from datetime import datetime, timedelta, timezone +from flask import Flask, current_app +from celery import states + +import logging +from logging import INFO +from _pytest.logging import LogCaptureFixture +from sqlalchemy.exc import SQLAlchemyError +from elasticsearch import ElasticsearchException from invenio_cache import current_cache +from weko_authors.tasks import export_all,import_author,check_is_import_available,import_id_prefix,import_affiliation_id,import_author_over_max,import_authors_from_temp_files,import_authors_for_over_max,write_result_temp_file,update_summary,prepare_display_status,prepare_success_msg,check_task_end,check_is_import_available,check_tmp_file_time_for_author,update_cache_data, \ +import_authors_from_temp_files, import_authors_for_over_max +from weko_workflow.utils import get_cache_data, delete_cache_data + +# from .config import WEKO_AUTHORS_IMPORT_CACHE_RESULT_OVER_MAX_FILE_PATH_KEY -from weko_authors.tasks import export_all,import_author,check_is_import_available # .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -15,34 +30,72 @@ def test_export_all(app,mocker): mocker.patch("weko_authors.tasks.save_export_url") mocker.patch("weko_authors.tasks.export_authors",return_value="test_url.txt") - result = export_all() + result = export_all('author_db') assert result == "test_url.txt" mocker.patch("weko_authors.tasks.export_authors",return_value=None) - result = export_all() + result = export_all('id_prefix') assert result == None mocker.patch("weko_authors.tasks.export_authors",side_effect=Exception) - result = export_all() + result = export_all('XXXX') assert result == None + # def import_author(author): -# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_import_author -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp -def test_import_author(app): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_01_import_author -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_01_import_author(app): with patch("weko_authors.tasks.import_author_to_system"): - result = import_author("test author") + result = import_author({"status":"", "weko_id":"", "current_weko_id":""}, True) assert result["status"] == "SUCCESS" - - with patch("weko_authors.tasks.import_author_to_system",side_effect=Exception({"error_id":"1"})): - result = import_author("test author") - assert result["status"] == "FAILURE" - assert result["error_id"] == "1" - with patch("weko_authors.tasks.import_author_to_system",side_effect=KeyError("is_deleted")): - result = import_author("test author") - assert result["status"] == "FAILURE" + + +# def import_author(author): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_02_import_author -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_02_import_author(app, caplog: LogCaptureFixture): + with patch("weko_authors.tasks.import_author_to_system",side_effect=SQLAlchemyError("SQLAlchemyError")): + result = import_author({"status":"", "weko_id":"", "current_weko_id":""}, True) + info_logs = [record for record in caplog.record_tuples if record[1] == logging.ERROR] + expected = [('testapp', logging.ERROR, 'SQLAlchemyError')] * 6 + assert info_logs == expected + assert result["status"] == "FAILURE" + + +# def import_author(author): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_03_import_author -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_03_import_author(app, caplog: LogCaptureFixture): + with patch("weko_authors.tasks.import_author_to_system",side_effect=ElasticsearchException("ElasticsearchException")): + result = import_author({"status":"", "weko_id":"", "current_weko_id":""}, True) + info_logs = [record for record in caplog.record_tuples if record[1] == logging.ERROR] + expected = [('testapp', logging.ERROR, 'ElasticsearchException')] * 6 + assert info_logs == expected + assert result["status"] == "FAILURE" + + +# def import_author(author): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_04_import_author -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_04_import_author(app, caplog: LogCaptureFixture): + with patch("weko_authors.tasks.import_author_to_system",side_effect=TimeoutError("TimeoutError")): + result = import_author({"status":"", "weko_id":"", "current_weko_id":""}, True) + info_logs = [record for record in caplog.record_tuples if record[1] == logging.ERROR] + expected = [('testapp', logging.ERROR, 'TimeoutError')] * 6 + assert info_logs == expected + assert result["status"] == "FAILURE" + + +# def import_author(author): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_05_import_author -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_05_import_author(app, caplog: LogCaptureFixture): + with patch("weko_authors.tasks.import_author_to_system",side_effect=TimeoutError({"error_id": 123, "message": "An error occurred"})): + result = import_author({"status":"", "weko_id":"", "current_weko_id":""}, True) + info_logs = [record for record in caplog.record_tuples if record[1] == logging.ERROR] + expected = [('testapp', logging.ERROR, "{'error_id': 123, 'message': 'An error occurred'}")] * 6 + assert info_logs == expected + assert result["status"] == "FAILURE" + # def check_is_import_available(group_task_id=None): -# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_check_is_import_available -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_check_is_import_available -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_check_is_import_available(app,mocker): cache_key = "author_import_cache" class MockTask: @@ -116,4 +169,756 @@ def ping(self): current_cache.set(cache_key,{"group_task_id":"not_success_task2"}) result = check_is_import_available(2) assert result == {"is_available":False} - assert current_cache.get(cache_key) is not None \ No newline at end of file + assert current_cache.get(cache_key) is not None + + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::TestImportAuthorsFromTempFiles -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestImportAuthorsFromTempFiles: + @pytest.fixture + def mock_update_cache_data(self): + """Mock update_cache_data function.""" + with patch('weko_authors.tasks.update_cache_data') as mock: + yield mock + + @pytest.fixture + def mock_get_check_base_name(self): + """Mock get_check_base_name function.""" + with patch('weko_authors.tasks.get_check_base_name', return_value='test_check_base') as mock: + yield mock + + @pytest.fixture + def mock_import_authors_for_over_max(self): + """Mock import_authors_for_over_max function.""" + with patch('weko_authors.tasks.import_authors_for_over_max') as mock: + yield mock + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::TestImportAuthorsFromTempFiles::test_import_authors_from_temp_files_normal_case -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_import_authors_from_temp_files_normal_case(self, app2, mock_update_cache_data, mock_get_check_base_name, + mock_import_authors_for_over_max): + """ + 正常系 + 条件：一時ファイルが正常に存在し、著者データを正常に読み込める + 入力： + - reached_point: {"part_number": 1, "count": 2} + - max_part: 2 + 期待結果： + - 結果ファイルパスをキャッシュに保存 + - 一時ファイルから著者データを読み込み + - import_authors_for_over_maxが呼び出される + - 一時ファイルが削除される + """ + # テスト用データの準備 + reached_point = {"part_number": 1, "count": 2} + max_part = 2 + + # 一時ファイルの内容をモック + part1_data = [ + {"pk_id": 1, "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}], "status": "new", "weko_id": "1001", "current_weko_id": "1000"}, + {"pk_id": 2, "authorNameInfo": [{"familyName": "Smith", "firstName": "Jane"}], "status": "update", "weko_id": "1002", "current_weko_id": "1002"}, + {"pk_id": 3, "authorNameInfo": [{"familyName": "Brown", "firstName": "Bob"}], "status": "new", "weko_id": "1003", "current_weko_id": ""}, + {"pk_id": 4, "authorNameInfo": [{"familyName": "Lee", "firstName": "Alice"}], "status": "update", "weko_id": "1004", "current_weko_id": "1004"}, + ] + + part2_data = [ + {"pk_id": 5, "authorNameInfo": [{"familyName": "Lee", "firstName": "Alice"}], "status": "update", "weko_id": "1004", "current_weko_id": "1004"}, + ] + + # ファイル読み込みのモック + mock_file_data = { + '/data/test_check_base-part1': json.dumps(part1_data), + '/data/test_check_base-part2': json.dumps(part2_data) + } + + def mock_open_side_effect(*args, **kwargs): + file_path = args[0] + if file_path in mock_file_data: + mock = mock_open(read_data=mock_file_data[file_path]) + return mock(*args, **kwargs) + return mock_open()(*args, **kwargs) + + # os.removeをモック + with patch('os.remove') as mock_remove, \ + patch('builtins.open', side_effect=mock_open_side_effect): + + # 関数実行 + import_authors_from_temp_files(reached_point, max_part) + + # アサーション + # update_cache_dataが呼ばれたことを確認 + mock_update_cache_data.assert_called_once() + + # import_authors_for_over_maxが2回呼ばれたことを確認 + assert mock_import_authors_for_over_max.call_count == 2 + + # 一時ファイルの削除が呼ばれたことを確認 + mock_remove.assert_has_calls([ + call('/data/test_check_base-part1'), + call('/data/test_check_base-part2') + ]) + + def test_import_authors_from_temp_files_with_batch_size(self, app2, mock_update_cache_data, mock_get_check_base_name, + mock_import_authors_for_over_max): + """ + 正常系 + 条件：著者データがバッチサイズを超える場合 + 入力： + - reached_point: {"part_number": 1, "count": 0} + - max_part: 1 + - WEKO_AUTHORS_IMPORT_BATCH_SIZE: 2 + 期待結果： + - バッチサイズ（2）ごとにimport_authors_for_over_maxが呼び出される + """ + + # テスト用データの準備 + reached_point = {"part_number": 1, "count": 0} + max_part = 1 + + # 一時ファイルの内容をモック（5人の著者） + part1_data = [ + {"pk_id": 1, "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}], "status": "new", "weko_id": "1001", "current_weko_id": ""}, + {"pk_id": 2, "authorNameInfo": [{"familyName": "Smith", "firstName": "Jane"}], "status": "update", "weko_id": "1002", "current_weko_id": "1002"}, + {"pk_id": 3, "authorNameInfo": [{"familyName": "Brown", "firstName": "Bob"}], "status": "new", "weko_id": "1003", "current_weko_id": ""}, + {"pk_id": 4, "authorNameInfo": [{"familyName": "Lee", "firstName": "Alice"}], "status": "update", "weko_id": "1004", "current_weko_id": "1004"}, + {"pk_id": 5, "authorNameInfo": [{"familyName": "Wang", "firstName": "Chen"}], "status": "new", "weko_id": "1005", "current_weko_id": "", "errors":["error"]} + ] + + # ファイル読み込みのモック + mock_file_data = { + '/data/test_check_base-part1': json.dumps(part1_data) + } + + def mock_open_side_effect(*args, **kwargs): + file_path = args[0] + if file_path in mock_file_data: + mock = mock_open(read_data=mock_file_data[file_path]) + return mock(*args, **kwargs) + return mock_open()(*args, **kwargs) + + # os.removeをモック + with patch('os.remove') as mock_remove, \ + patch('builtins.open', side_effect=mock_open_side_effect): + + # 関数実行 + import_authors_from_temp_files(reached_point, max_part) + + # アサーション + # import_authors_for_over_maxが呼ばれたことを確認 + assert mock_import_authors_for_over_max.call_count == 1 + + # 最初のバッチ（0, 1） + mock_import_authors_for_over_max.assert_any_call(part1_data[0:4]) + + def test_import_authors_from_temp_files_file_not_found(self, app2, mock_update_cache_data, mock_get_check_base_name, + mock_import_authors_for_over_max): + """ + 異常系 + 条件：一時ファイルが存在しない場合 + 入力： + - reached_point: {"part_number": 1, "count": 0} + - max_part: 1 + 期待結果： + - FileNotFoundErrorが発生する + - エラーログが記録される + """ + # テスト用データの準備 + reached_point = {"part_number": 2, "count": 0} + max_part = 1 + # 一時ファイルの内容をモック（5人の著者） + part1_data = [ + {"pk_id": 1, "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}], "status": "new", "weko_id": "1001", "current_weko_id": ""}, + {"pk_id": 2, "authorNameInfo": [{"familyName": "Smith", "firstName": "Jane"}], "status": "update", "weko_id": "1002", "current_weko_id": "1002"}, + {"pk_id": 3, "authorNameInfo": [{"familyName": "Brown", "firstName": "Bob"}], "status": "new", "weko_id": "1003", "current_weko_id": ""}, + {"pk_id": 4, "authorNameInfo": [{"familyName": "Lee", "firstName": "Alice"}], "status": "update", "weko_id": "1004", "current_weko_id": "1004"}, + {"pk_id": 5, "authorNameInfo": [{"familyName": "Wang", "firstName": "Chen"}], "status": "new", "weko_id": "1005", "current_weko_id": "", "errors":["error"]} + ] + + # ファイル読み込みのモック + mock_file_data = { + '/data/test_check_base-part1': json.dumps(part1_data) + } + + def mock_open_side_effect(*args, **kwargs): + file_path = args[0] + if file_path in mock_file_data: + mock = mock_open(read_data=mock_file_data[file_path]) + return mock(*args, **kwargs) + return mock_open()(*args, **kwargs) + + # os.removeをモック + with patch('os.remove', side_effect=FileNotFoundError("File not found")) as mock_remove, \ + patch('builtins.open', side_effect=mock_open_side_effect): + + import_authors_from_temp_files(reached_point, max_part) + + def test_import_authors_from_temp_files_invalid_json(self, app2, mock_update_cache_data, mock_get_check_base_name, + mock_import_authors_for_over_max): + """ + 異常系 + 条件：一時ファイルのJSONが不正な場合 + 入力： + - reached_point: {"part_number": 1, "count": 0} + - max_part: 1 + 期待結果： + - JSONDecodeErrorが発生する + - エラーログが記録される + """ + # テスト用データの準備 + reached_point = {"part_number": 1, "count": 0} + max_part = 1 + + # 不正なJSONをシミュレート + invalid_json = "{ invalid json }" + + # ファイル読み込みのモック + mock_file_data = { + '/data/test_check_base-part1': invalid_json + } + + def mock_open_side_effect(*args, **kwargs): + file_path = args[0] + if file_path in mock_file_data: + mock = mock_open(read_data=mock_file_data[file_path]) + return mock(*args, **kwargs) + return mock_open()(*args, **kwargs) + + # os.removeをモック + with patch('os.remove') as mock_remove, \ + patch('builtins.open', side_effect=mock_open_side_effect): + + # 関数実行してエラーを検証 + with pytest.raises(json.JSONDecodeError): + import_authors_from_temp_files(reached_point, max_part) + + # 削除は呼ばれないはず + mock_remove.assert_not_called() + + def test_import_authors_from_temp_files_file_deletion_error(self, app2, mock_update_cache_data, mock_get_check_base_name, + mock_import_authors_for_over_max): + """ + 異常系 + 条件：一時ファイルの削除に失敗する場合 + 入力： + - reached_point: {"part_number": 1, "count": 0} + - max_part: 1 + 期待結果： + - エラーログが記録される + - 処理は続行される + """ + # テスト用データの準備 + reached_point = {"part_number": 1, "count": 0} + max_part = 1 + + # 一時ファイルの内容をモック + part1_data = [ + {"pk_id": 1, "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}], "status": "new", "weko_id": "1001", "current_weko_id": ""} + ] + + # ファイル読み込みのモック + mock_file_data = { + '/data/test_check_base-part1': json.dumps(part1_data) + } + + def mock_open_side_effect(*args, **kwargs): + file_path = args[0] + if file_path in mock_file_data: + mock = mock_open(read_data=mock_file_data[file_path]) + return mock(*args, **kwargs) + return mock + + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::TestImportAuthorsForOverMax -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestImportAuthorsForOverMax: + """Test class for import_authors_for_over_max function.""" + + @pytest.fixture + def mock_cache(self): + """Mock the current_cache.""" + with patch('weko_authors.tasks.current_cache') as mock_cache: + mock_cache.get.return_value = "/temp/result_file.tsv" + yield mock_cache + + @pytest.fixture + def mock_get_cache_data(self): + """Mock the get_cache_data function.""" + with patch('weko_authors.tasks.get_cache_data') as mock_get: + mock_get.return_value = {"success_count": 5, "failure_count": 2} + yield mock_get + + @pytest.fixture + def mock_update_summary(self): + """Mock the update_cache_data function.""" + with patch('weko_authors.tasks.update_summary') as mock_update: + yield mock_update + + @pytest.fixture + def mock_csv_writer(self): + """Mock csv writer.""" + mock_writer = MagicMock() + with patch('weko_authors.tasks.open', mock_open()) as mock_file, \ + patch('csv.writer', return_value=mock_writer) as _: + yield mock_writer + + @pytest.fixture + def mock_group(self): + """Mock celery group.""" + with patch('weko_authors.tasks.group') as mock_group: + mock_task = MagicMock() + mock_task.children = [] + mock_group.return_value.apply_async.return_value = mock_task + yield mock_group, mock_task + + @pytest.fixture + def mock_check_task_end(self): + """Mock check_task_end function.""" + with patch('weko_authors.tasks.check_task_end') as mock_check: + yield mock_check + + def test_import_authors_for_over_max_success(self, app2, mock_cache, + mock_get_cache_data, mock_update_summary, + mock_csv_writer, mock_group, mock_check_task_end): + """ + 正常系 + 条件：複数の著者情報が渡されて、全てインポート成功する場合 + 入力：有効な著者情報の配列 + 期待結果： + - グループタスクが作成・実行される + - タスクIDが保存される + - 結果ファイルが作成される + - サマリーが更新される + """ + # Setup + mock_group_obj, mock_task = mock_group + author1 = { + "pk_id": 1, + "current_weko_id": None, + "weko_id": "author1", + "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}], + "status": "new" + } + author2 = { + "pk_id": 2, + "current_weko_id": "old_author2", + "weko_id": "author2", + "authorNameInfo": [{"familyName": "Smith", "firstName": "Jane"}], + "status": "update" + } + authors = [author1, author2] + + # Setup task children and results + task1_mock = MagicMock() + task1_mock.task_id = "task1" + task1_mock.result = { + 'start_date': '2025-03-16 10:00:00', + 'end_date': '2025-03-16 10:00:05', + 'status': states.SUCCESS + } + + task2_mock = MagicMock() + task2_mock.task_id = "task2" + task2_mock.result = { + 'start_date': '2025-03-16 10:00:00', + 'end_date': '2025-03-16 10:00:05', + 'status': states.SUCCESS + } + + mock_task.children = [task1_mock, task2_mock] + + with patch('weko_authors.tasks.import_author.AsyncResult') as mock_async_result: + mock_async_result.side_effect = [task1_mock, task2_mock] + + # Execute + import_authors_for_over_max(authors) + + # Verify + assert mock_group_obj.call_count == 1 + mock_check_task_end.assert_called_once() + mock_update_summary.assert_called() + + # Check that we wrote to the result file + assert mock_csv_writer.writerow.call_count == 2 + + # Check task results were processed and records were forgotten + assert task1_mock.forget.call_count == 1 + assert task2_mock.forget.call_count == 1 + + def test_import_authors_for_over_max_partial_failure(self, app2, mock_cache, + mock_get_cache_data, mock_update_summary, + mock_csv_writer, mock_group, mock_check_task_end): + """ + 正常系 + 条件：複数の著者情報が渡されて、一部がインポート失敗する場合 + 入力：有効な著者情報の配列 (1つ成功、1つ失敗) + 期待結果： + - 成功・失敗の両方の結果が記録される + - エラーIDが記録される + - サマリーが正確に更新される + """ + # Setup + mock_group_obj, mock_task = mock_group + author1 = { + "pk_id": 1, + "current_weko_id": None, + "weko_id": "author1", + "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}], + "status": "new" + } + author2 = { + "pk_id": 2, + "current_weko_id": "old_author2", + "weko_id": "author2", + "authorNameInfo": [{"familyName": "Smith", "firstName": "Jane"}], + "status": "update" + } + authors = [author1, author2] + + # Setup task children and results + task1_mock = MagicMock() + task1_mock.task_id = "task1" + task1_mock.result = { + 'start_date': '2025-03-16 10:00:00', + 'end_date': '2025-03-16 10:00:05', + 'status': states.SUCCESS + } + + task2_mock = MagicMock() + task2_mock.task_id = "task2" + task2_mock.result = { + 'start_date': '2025-03-16 10:00:00', + 'end_date': '2025-03-16 10:00:05', + 'status': states.FAILURE, + 'error_id': 'delete_author_link' + } + + mock_task.children = [task1_mock, task2_mock] + + with patch('weko_authors.tasks.import_author.AsyncResult') as mock_async_result: + mock_async_result.side_effect = [task1_mock, task2_mock] + + # Execute + import_authors_for_over_max(authors) + + # Verify + assert mock_group_obj.call_count == 1 + mock_check_task_end.assert_called_once() + + # Check that the summary was updated with correct counts + mock_update_summary.assert_called_with(1,1) + + def test_import_authors_for_over_max_timeout(self, app2, mock_cache, + mock_get_cache_data, mock_update_summary, + mock_csv_writer, mock_group, mock_check_task_end): + """ + 異常系 + 条件：タスクがタイムアウトする場合 + 入力：有効な著者情報の配列 + 期待結果： + - タイムアウトエラーが記録される + - 失敗としてカウントされる + """ + # Setup + mock_group_obj, mock_task = mock_group + author = { + "pk_id": 1, + "current_weko_id": None, + "weko_id": "author1", + "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}], + "status": "new" + } + authors = [author] + + # Setup task children with timeout (result=None) + task_mock = MagicMock() + task_mock.task_id = "task1" + task_mock.result = None # Simulate timeout with no result + + mock_task.children = [task_mock] + + with patch('weko_authors.tasks.import_author.AsyncResult') as mock_async_result, \ + patch('weko_authors.tasks.datetime') as mock_datetime: + + mock_datetime.now.return_value.strftime.return_value = '2025-03-16 10:00:00' + mock_async_result.return_value = task_mock + + # Execute + import_authors_for_over_max(authors) + + # Verify + mock_check_task_end.assert_called_once() + + # Check that we wrote to the result file with timeout error + mock_csv_writer.writerow.assert_called_once() + + # Check that the summary was updated with failure count + mock_update_summary.assert_called_with(0,1) + + def test_import_authors_for_over_max_no_summary(self, app2, mock_cache, + mock_get_cache_data, mock_update_summary, + mock_csv_writer, mock_group, mock_check_task_end): + """ + 正常系 + 条件：サマリーデータが存在しない場合 + 入力：有効な著者情報の配列 + 期待結果： + - 新しいサマリーが作成される + - 正確なカウントが記録される + """ + # Setup - returns None for summary data + mock_get_cache_data.return_value = None + + mock_group_obj, mock_task = mock_group + author = { + "pk_id": 1, + "current_weko_id": None, + "weko_id": "author1", + "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}, + {"familyName": "山田", "firstName": "太郎"}, + {"familyName": "", "firstName": ""}], + "status": "new" + } + authors = [author] + + task_mock = MagicMock() + task_mock.task_id = "task1" + task_mock.result = { + 'start_date': '2025-03-16 10:00:00', + 'end_date': '2025-03-16 10:00:05', + 'status': states.SUCCESS + } + + mock_task.children = [task_mock] + + with patch('weko_authors.tasks.import_author.AsyncResult') as mock_async_result: + mock_async_result.return_value = task_mock + + # Execute + import_authors_for_over_max(authors) + + # Verify + # Check that a new summary was created + mock_update_summary.assert_called_with(1,0) + + def test_import_authors_for_over_max_for_cover(self, app2, mock_cache, + mock_get_cache_data, mock_update_summary, + mock_csv_writer, mock_group, mock_check_task_end): + """ + 正常系 + 条件：AsyncResultのstatusがPENDINGの場合 + 入力：有効な著者情報の配列 + 期待結果： + - 正確なカウントが記録される + """ + # Setup - returns None for summary data + mock_get_cache_data.return_value = None + + mock_group_obj, mock_task = mock_group + author = { + "pk_id": 1, + "current_weko_id": None, + "weko_id": "author1", + "authorNameInfo": [{"familyName": "Doe", "firstName": "John"}, + {"familyName": "山田", "firstName": "太郎"}, + {"familyName": "", "firstName": ""}], + "status": "new" + } + authors = [author] + + task_mock = MagicMock() + task_mock.task_id = "task1" + task_mock.result = { + 'start_date': '2025-03-16 10:00:00', + 'end_date': '2025-03-16 10:00:05', + 'status': states.PENDING + } + + mock_task.children = [task_mock] + + with patch('weko_authors.tasks.import_author.AsyncResult') as mock_async_result: + mock_async_result.return_value = task_mock + + # Execute + import_authors_for_over_max(authors) + + # Verify + # Check that a new summary was created + mock_update_summary.assert_called_with(0,0) + + +# def import_id_prefix(prefix): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_import_id_prefix -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_import_id_prefix(app, caplog: LogCaptureFixture): + result = import_id_prefix(None) + assert result['status'] == 'SUCCESS' + assert 'start_date' in result + assert 'end_date' in result + + with patch('weko_authors.tasks.import_id_prefix_to_system', side_effect=ValueError({"error_id": 123, "message": "DB upload failed"})): + # mock_func.side_effect = Exception("Mocked exception") + # with pytest.raises(Exception) as ex: + result = import_id_prefix(None) + info_logs = [record for record in caplog.record_tuples if record[1] == logging.ERROR] + assert [('testapp', logging.ERROR, "{'error_id': 123, 'message': 'DB upload failed'}")] == info_logs + assert result['status'] == 'FAILURE' + + +# def import_affiliation_id(affiliation_id): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_import_affiliation_id -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_import_affiliation_id(app, caplog: LogCaptureFixture): + result = import_affiliation_id(None) + assert result['status'] == 'SUCCESS' + assert 'start_date' in result + assert 'end_date' in result + + with patch('weko_authors.tasks.import_affiliation_id_to_system', side_effect=ValueError({"error_id": 123, "message": "DB upload failed"})): + result = import_affiliation_id(None) + info_logs = [record for record in caplog.record_tuples if record[1] == logging.ERROR] + assert [('testapp', logging.ERROR, "{'error_id': 123, 'message': 'DB upload failed'}")] == info_logs + assert result['status'] == 'FAILURE' + + +# def import_author_over_max(reached_point, task_ids ,max_part): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_import_author_over_max -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_import_author_over_max(app, caplog: LogCaptureFixture): + reached_points = {"part_number": 101, "count": 3} + task_ids = ["aaa","bbb","ccc"] + + with patch('weko_authors.tasks.check_task_end', return_value=0), \ + patch("weko_authors.tasks.import_authors_from_temp_files") as mock_temp: + result = import_author_over_max(reached_points, task_ids, 5) + assert result['status'] == 'SUCCESS' + mock_temp.assert_called() + + with patch('weko_authors.tasks.check_task_end', return_value=0): + with patch("weko_authors.tasks.import_authors_from_temp_files", side_effect=ValueError({"error_id": 123, "message": "file not found"})) as mock_temp: + info_logs = [record for record in caplog.record_tuples if record[1] == logging.ERROR] + result = import_author_over_max(reached_points, task_ids, 5) + [('testapp', logging.ERROR, "{'error_id': 123, 'message': 'file not found'}")] == info_logs + assert result['status'] == 'FAILURE' + + +# def write_result_temp_file(result): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_write_result_temp_file -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_write_result_temp_file(app): + # with patch('weko_authors.tasks.write_result_temp_file', result_file_path): + # with patch('weko_authors.tasks.current_cache.get') as mock_getenv: + # mock_getenv.return_value = result_file_path + test = [{ + "start_date": 2025-3-10, + "end_date": 2025-3-14, + "weko_id": 1, + "full_name": "kimura,shinji", + "type": "new", + "status": 'SUCCESS', + "error_id": "delete_author_link" + }] + with patch("weko_authors.tasks.open") as mock_writer: + update_cache_data(current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_OVER_MAX_FILE_PATH_KEY"], "/data/test_over_max") + write_result_temp_file(test) + mock_writer.assert_called() + + with pytest.raises(Exception): + write_result_temp_file(test) + + +# def update_summary(success_count, failure_count): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_update_summary -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_update_summary(app): + delete_cache_data(current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY"]) + update_summary(2, 2) + summary = get_cache_data(current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY"]) + assert summary["success_count"] == 2 + assert summary["failure_count"] == 2 + + update_summary(2, 2) + summary = get_cache_data(current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY"]) + assert summary["success_count"] == 4 + assert summary["failure_count"] == 4 + + +# def prepare_display_status(status, type, error_id): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_prepare_display_status -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_prepare_display_status(): + result = prepare_display_status('SUCCESS', 'new', None) + assert result == 'Register Success' + + result = prepare_display_status('FAILURE', type, 'delete_author_link') + assert result == 'Error: The author is linked to items and cannot be deleted.' + + result = prepare_display_status('FAILURE', type, 'other_error') + assert result == 'Error: Failed to import.' + + +# def prepare_success_msg(type): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_prepare_success_msg -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_prepare_success_msg(): + result = prepare_success_msg('new') + assert result == 'Register Success' + + result = prepare_success_msg('update') + assert result == 'Update Success' + + result = prepare_success_msg('deleted') + assert result == 'Delete Success' + + result = prepare_success_msg('none') + assert result == '' + + +# def check_task_end(task_ids): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_check_task_end -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_check_task_end(app): + data = ["aaa","bbb","ccc"] + mock_task1 = MagicMock() + mock_task1.result = {'end_date': '2023-01-01'} + + mock_task2 = MagicMock() + mock_task2.result = {'end_date': '2023-01-01'} + + mock_task3 = MagicMock() + mock_task3.result = {'end_date': '2023-01-01'} + with patch('weko_authors.tasks.import_author.AsyncResult', side_effect=[mock_task1, mock_task2, mock_task3]),\ + patch("weko_authors.tasks.sleep") as mock_sleep: + result = check_task_end(data) + assert result == 0 + mock_sleep.assert_not_called() + with patch("weko_authors.tasks.import_author.AsyncResult",return_value=MagicMock(result = "")),\ + patch("weko_authors.tasks.sleep") as mock_sleep: + result = check_task_end(data) + mock_sleep.assert_called() + + +# def check_tmp_file_time_for_author(): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_tasks.py::test_check_tmp_file_time_for_author -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_check_tmp_file_time_for_author(app, caplog: LogCaptureFixture): + mock_file_path = '/mock/path/to/file' + mock_current_time = datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + with patch('glob.glob', return_value=[mock_file_path]), \ + patch('os.path.getmtime', return_value=(mock_current_time - timedelta(seconds=3600)).timestamp()), \ + patch('os.remove') as mock_remove,\ + patch('os.rmdir') as mock_rmdir,\ + patch('os.path.exists'),\ + patch('os.listdir', return_value=[]): + + check_tmp_file_time_for_author() + mock_remove.assert_called() + mock_rmdir.assert_called() + + mock_current_time = datetime.now(timezone.utc) + with patch('glob.glob', return_value=[mock_file_path]), \ + patch('os.path.getmtime', return_value=(mock_current_time - timedelta(seconds=3600)).timestamp()), \ + patch('os.remove') as mock_remove,\ + patch('os.rmdir') as mock_rmdir,\ + patch('os.path.exists', result_value = True),\ + patch('os.listdir', return_value=["a"]): + + check_tmp_file_time_for_author() + mock_remove.assert_not_called() + mock_rmdir.assert_not_called() + + mock_current_time = datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + with patch('glob.glob', return_value=[mock_file_path]), \ + patch('os.path.getmtime', return_value=(mock_current_time - timedelta(seconds=3600)).timestamp()), \ + patch('os.remove', side_effect=OSError) as mock_remove,\ + patch('os.listdir', return_value=["a"]): + caplog.set_level(logging.ERROR) + + check_tmp_file_time_for_author() diff --git a/modules/weko-authors/tests/test_utils.py b/modules/weko-authors/tests/test_utils.py index 50f66e050d..111caa749e 100644 --- a/modules/weko-authors/tests/test_utils.py +++ b/modules/weko-authors/tests/test_utils.py @@ -1,12 +1,17 @@ from os.path import dirname, join import pytest -from mock import patch +import copy +from mock import patch, MagicMock, Mock from flask import current_app +import datetime +from unittest.mock import mock_open from invenio_indexer.api import RecordIndexer from invenio_cache import current_cache +import csv +from weko_authors.models import Authors from weko_authors.config import WEKO_AUTHORS_FILE_MAPPING from weko_authors.utils import ( get_author_prefix_obj, @@ -29,7 +34,22 @@ set_record_status, flatten_authors_mapping, import_author_to_system, - get_count_item_link + get_count_item_link, + count_authors, + update_cache_data, + write_tmp_part_file, + unpackage_and_check_import_file_for_prefix, + get_check_base_name, + get_check_result, + handle_check_consistence_with_mapping_for_prefix, + check_import_data_for_prefix, + validate_import_data_for_prefix, + import_id_prefix_to_system, + import_affiliation_id_to_system, + band_check_file_for_user, + prepare_import_data, + write_to_tempfile, + create_result_file_for_user ) # .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -128,32 +148,7 @@ def test_save_export_url(app): current_cache.delete("weko_authors_exported_url") -# def export_authors(): -# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_export_authors -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp -def test_export_authors(app,authors,location,file_instance,mocker): - mocker.patch("weko_authors.utils.WekoAuthors.get_all",return_value=authors) - scheme_info={"1":{"scheme":"WEKO","url":None},"2":{"scheme":"ORCID","url":"https://orcid.org/##"}} - mocker.patch("weko_authors.utils.WekoAuthors.get_identifier_scheme_info",return_value=scheme_info) - header = ["#pk_id","authorNameInfo[0].familyName","authorNameInfo[0].firstName","authorNameInfo[0].language","authorNameInfo[0].nameFormat","authorNameInfo[0].nameShowFlg","authorIdInfo[0].idType","authorIdInfo[0].authorId","authorIdInfo[0].authorIdShowFlg","emailInfo[0].email","is_deleted"] - label_en=["#WEKO ID","Family Name[0]","Given Name[0]","Language[0]","Name Format[0]","Name Display[0]","Identifier Scheme[0]","Identifier[0]","Identifier Display[0]","Mail Address[0]","Delete Flag"] - label_jp=["#WEKO ID","姓[0]","名[0]","言語[0]","フォーマット[0]","姓名・言語 表示／非表示[0]","外部著者ID 識別子[0]","外部著者ID[0]","外部著者ID 表示／非表示[0]","メールアドレス[0]","削除フラグ"] - row_data = [["1","テスト","太郎","ja","familyNmAndNm","Y","ORCID","1234","Y","test.taro@test.org",""], - ["2","test","smith","en","familyNmAndNm","Y","ORCID","5678","Y","test.smith@test.org",""]] - mocker.patch("weko_authors.utils.WekoAuthors.prepare_export_data",return_value=(header,label_en,label_jp,row_data)) - with patch("weko_authors.utils.get_export_url",return_value={}): - result = export_authors() - assert result - - current_app.config.update(WEKO_ADMIN_OUTPUT_FORMAT="csv") - cache_url = {"file_uri":"/var/tmp/test_dir"} - with patch("weko_authors.utils.get_export_url",return_value=cache_url): - result = export_authors() - assert result == "/var/tmp/test_dir" - - # raise Exception - with patch("weko_authors.utils.WekoAuthors.get_all", side_effect=Exception("test_error")): - result = export_authors() - assert result == None + # def check_import_data(file_name: str, file_content: str): # .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_check_import_data -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -183,35 +178,247 @@ def test_check_import_data(app,mocker): "emailInfo[0].email", "is_deleted" ] + current_cache.set("authors_import_user_file_key","var/tmp/authors_import") mocker.patch("weko_authors.utils.flatten_authors_mapping",return_value=(mapping_all,mapping_ids)) - file_data = [{'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '1234', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': ''}, {'pk_id': '2', 'authorNameInfo': [{'familyName': 'test', 'firstName': 'smith', 'language': 'en', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '5678', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.smith@test.org'}], 'is_deleted': ''}] - mocker.patch("weko_authors.utils.unpackage_and_check_import_file",return_value=file_data) + mocker.patch("weko_authors.utils.unpackage_and_check_import_file",return_value=1) return_validate = [{'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '1234', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': '', 'status': 'update'}, {'pk_id': '2', 'authorNameInfo': [{'familyName': 'test', 'firstName': 'smith', 'language': 'en', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '5678', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.smith@test.org'}], 'is_deleted': '', 'status': 'update'}] mocker.patch("weko_authors.utils.validate_import_data",return_value=return_validate) + mock_json_data = '{"test_id": "1000"}' + mocker.patch("builtins.open", mock_open(read_data=mock_json_data)) + mocker.patch("os.remove") + file_name = "testfile.tsv" + test = { + "list_import_data": {"test_id": "1000"}, + "max_page": 1, + "counts": { + "num_total": 2, + "num_new": 0, + "num_update": 2, + "num_delete": 0, + "num_error": 0, + }, + } + result = check_import_data(file_name) + assert result == test + current_cache.set("authors_import_user_file_key","var/tmp/authors_import") + mocker.patch("weko_authors.utils.flatten_authors_mapping",return_value=(mapping_all,mapping_ids)) + mocker.patch("weko_authors.utils.unpackage_and_check_import_file",return_value=1) + return_validate = [{'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '1234', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': '', 'status': 'update'}, {'pk_id': '2', 'authorNameInfo': [{'familyName': 'test', 'firstName': 'smith', 'language': 'en', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '5678', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.smith@test.org'}], 'is_deleted': '', 'status': 'update'}] + mocker.patch("weko_authors.utils.validate_import_data",return_value=return_validate) + mock_json_data = '{"test_id": "1000"}' + mocker.patch("builtins.open", mock_open(read_data=mock_json_data)) + mocker.patch("os.remove") + file_name = "valid_file.csv" + test = { + "list_import_data": {"test_id": "1000"}, + "max_page": 1, + "counts": { + "num_total": 2, + "num_new": 0, + "num_update": 2, + "num_delete": 0, + "num_error": 0, + }, + } + result = check_import_data(file_name) + assert result == test - file_name = "testfile.tsv" - file_content = b"test_content==" - test = {'list_import_data': [{'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '1234', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': '', 'status': 'update'}, {'pk_id': '2', 'authorNameInfo': [{'familyName': 'test', 'firstName': 'smith', 'language': 'en', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '5678', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.smith@test.org'}], 'is_deleted': '', 'status': 'update'}]} - result = check_import_data(file_name,file_content) + current_cache.set("authors_import_user_file_key","var/tmp/authors_import") + mocker.patch("weko_authors.utils.flatten_authors_mapping",return_value=(mapping_all,mapping_ids)) + mocker.patch("weko_authors.utils.unpackage_and_check_import_file",return_value=5) + return_validate = [{'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '1234', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': '', 'status': 'update'}, {'pk_id': '2', 'authorNameInfo': [{'familyName': 'test', 'firstName': 'smith', 'language': 'en', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '5678', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.smith@test.org'}], 'is_deleted': '', 'status': 'update'}] + mocker.patch("weko_authors.utils.validate_import_data",return_value=return_validate) + mock_json_data = '{"test_id": "1000"}' + mocker.patch("builtins.open", mock_open(read_data=mock_json_data)) + mocker.patch("os.remove") + file_name = "multi_part_file.tsv" + test = { + "list_import_data": {"test_id": "1000"}, + "max_page": 5, + "counts": { + "num_total": 10, + "num_new": 0, + "num_update": 10, + "num_delete": 0, + "num_error": 0, + }, + } + result = check_import_data(file_name) assert result == test + + current_cache.set("authors_import_user_file_key","var/tmp/authors_import") + mocker.patch("weko_authors.utils.flatten_authors_mapping",return_value=(mapping_all,mapping_ids)) + mocker.patch("weko_authors.utils.unpackage_and_check_import_file",return_value=5) + return_validate = [ + { + "pk_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true", + } + ], + "authorIdInfo": [{"idType": 2, "authorId": "1234", "authorIdShowFlg": "true"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "new", + }, + { + "pk_id": "2", + "authorNameInfo": [ + { + "familyName": "test", + "firstName": "smith", + "language": "en", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true", + } + ], + "authorIdInfo": [{"idType": 2, "authorId": "5678", "authorIdShowFlg": "true"}], + "emailInfo": [{"email": "test.smith@test.org"}], + "is_deleted": "", + "status": "update", + }, + { + "pk_id": "3", + "authorNameInfo": [ + { + "familyName": "test", + "firstName": "smith", + "language": "en", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true", + } + ], + "authorIdInfo": [{"idType": 2, "authorId": "5678", "authorIdShowFlg": "true"}], + "emailInfo": [{"email": "test.smith@test.org"}], + "is_deleted": "", + "status": "deleted", + }, + ] + mocker.patch("weko_authors.utils.validate_import_data",return_value=return_validate) + mock_json_data = '{"test_id": "1000"}' + mocker.patch("builtins.open", mock_open(read_data=mock_json_data)) + mocker.patch("os.remove") + file_name = "multi_part_file.tsv" + test = { + "list_import_data": {"test_id": "1000"}, + "max_page": 5, + "counts": { + "num_total": 15, + "num_new": 5, + "num_update": 5, + "num_delete": 5, + "num_error": 0, + }, + } + result = check_import_data(file_name) + assert result == test + + current_cache.set("authors_import_user_file_key","var/tmp/authors_import") + mocker.patch("weko_authors.utils.flatten_authors_mapping",return_value=(mapping_all,mapping_ids)) + mocker.patch("weko_authors.utils.unpackage_and_check_import_file",return_value=1) + return_validate = [ + { + "pk_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true", + } + ], + "authorIdInfo": [{"idType": 2, "authorId": "1234", "authorIdShowFlg": "true"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "new", + "errors": ["Existing error"] + }, + { + "pk_id": "2", + "authorNameInfo": [ + { + "familyName": "test", + "firstName": "smith", + "language": "en", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true", + } + ], + "authorIdInfo": [{"idType": 2, "authorId": "5678", "authorIdShowFlg": "true"}], + "emailInfo": [{"email": "test.smith@test.org"}], + "is_deleted": "", + "status": "update", + "errors": ["Existing error"] + }, + { + "pk_id": "3", + "authorNameInfo": [ + { + "familyName": "test", + "firstName": "smith", + "language": "en", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "true", + } + ], + "authorIdInfo": [{"idType": 2, "authorId": "5678", "authorIdShowFlg": "true"}], + "emailInfo": [{"email": "test.smith@test.org"}], + "is_deleted": "", + "status": "delete", + "errors": ["Existing error"] + }, + ] + mocker.patch("weko_authors.utils.validate_import_data",return_value=return_validate) + mock_json_data = '{"test_id": "1000"}' + mocker.patch("builtins.open", mock_open(read_data=mock_json_data)) + mocker.patch("os.remove") + file_name = "valid_file.csv" + test = { + "list_import_data": {"test_id": "1000"}, + "max_page": 1, + "counts": { + "num_total": 3, + "num_new": 0, + "num_update": 0, + "num_delete": 0, + "num_error": 3, + }, + } + result = check_import_data(file_name) + assert result == test + + current_cache.delete("authors_import_user_file_key") + mocker.patch("weko_authors.utils.flatten_authors_mapping",return_value=(mapping_all,mapping_ids)) + mock_json_data = '{"test_id": "1000"}' + mocker.patch("builtins.open", mock_open(read_data=mock_json_data)) + mocker.patch("os.remove", side_effect=Exception) + file_name = "testfile.tsv" + mock_logger = mocker.patch("weko_authors.utils.current_app.logger") + result = check_import_data(file_name) + mock_logger.error.assert_called() # raise Exception with args[0] is dict with patch("weko_authors.utils.flatten_authors_mapping",side_effect=Exception({"error_msg":"test_error"})): - result = check_import_data(file_name,file_content) + result = check_import_data(file_name) assert result == {"error":"test_error"} # raise Exception with args[0] is not dict with patch("weko_authors.utils.flatten_authors_mapping",side_effect=AttributeError("test_error")): - result = check_import_data(file_name,file_content) + result = check_import_data(file_name) assert result == {"error":"Internal server error"} # raise UnicodeDecodeError with patch("weko_authors.utils.flatten_authors_mapping",side_effect=UnicodeDecodeError("utf-8",b"test",0,1,"test_reason")): - result = check_import_data(file_name,file_content) + result = check_import_data(file_name) assert result == {"error":"test_reason"} - # def getEncode(filepath): # .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_getEncode -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_getEncode(): @@ -239,7 +446,7 @@ def mock_open(path, encoding=None): # def unpackage_and_check_import_file(file_format, file_name, temp_file, mapping_ids): # .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_unpackage_and_check_import_file -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp -def test_unpackage_and_check_import_file(app): +def test_unpackage_and_check_import_file(app, mocker): file_format="tsv" file_name="test_file.txt" mapping_ids = ["pk_id", @@ -268,10 +475,10 @@ def test_unpackage_and_check_import_file(app): assert e.value.args[0] == {"error_msg":"Specified item does not consistency with DB item.
exist_not_consistent_value"} # nomal + mocker.patch("weko_authors.utils.write_tmp_part_file") temp_file=join(dirname(__file__),"data/test_files/import_data.tsv") result = unpackage_and_check_import_file(file_format,file_name,temp_file,mapping_ids) - test = [{'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '1234', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': ''}, {'pk_id': '2', 'authorNameInfo': [{'familyName': 'test', 'firstName': 'smith', 'language': 'en', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '5678', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.smith@test.org'}], 'is_deleted': ''}] - assert result == test + assert result == 1 # data_parse_metadata is None with patch("weko_search_ui.utils.parse_to_json_form", return_value=[]): @@ -292,22 +499,39 @@ def test_unpackage_and_check_import_file(app): assert e.value.args[0] == {"error_msg":"There is no data to import."} # file format = csv + mocker.patch.object(current_app.config, "get", return_value=2) + mocker.patch("weko_authors.utils.write_tmp_part_file") temp_file=join(dirname(__file__),"data/test_files/import_data.csv") result = unpackage_and_check_import_file("csv",file_name,temp_file,mapping_ids) - test = [{'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '1234', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': ''}, {'pk_id': '2', 'authorNameInfo': [{'familyName': 'test', 'firstName': 'smith', 'language': 'en', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '5678', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.smith@test.org'}], 'is_deleted': ''}] - assert result == test - + assert result == 1 + -# def validate_import_data(file_format, file_data, mapping_ids, mapping): +# def validate_import_data(file_format, file_data, mapping_ids, mapping, list_import_id): # .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_validate_import_data -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_validate_import_data(authors_prefix_settings,mocker): mocker.patch("weko_authors.utils.WekoAuthors.get_author_for_validation",return_value=({"1":True,"2":True},{"2":{"1234":["1"],"5678":["2"]}})) file_format = "tsv" file_data = [ - {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '1234', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': ''}, - {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '1234', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': ''}, - ] + { + "pk_id": "1", + "weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [ + {"idType": "ORCID", "authorId": "1234", "authorIdShowFlg": "Y"} + ], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + } + ] mapping_ids = ["pk_id", "authorNameInfo[0].familyName", "authorNameInfo[0].firstName", @@ -320,51 +544,478 @@ def test_validate_import_data(authors_prefix_settings,mocker): "emailInfo[0].email", "is_deleted" ] - mappings = [ - {"key":"pk_id","label":{"en":"WEKO ID","jp":"WEKO ID"},"mask":{},"validation":{},"autofill":""}, - {"key":"authorNameInfo[0].familyName","label":{"en":"Family Name","jp":"姓"},"mask":{},"validation":{},"autofill":""}, - {"key":"authorNameInfo[0].firstName","label":{"en":"Given Name","jp":"名"},"mask":{},"validation":{},"autofill":""}, - {"key":"authorNameInfo[0].language","label":{"en":"Language","jp":"言語"},"mask":{},"validation":{'map': ['ja', 'ja-Kana', 'en', 'fr','it', 'de', 'es', 'zh-cn', 'zh-tw','ru', 'la', 'ms', 'eo', 'ar', 'el', 'ko']},"autofill":""}, - {"key":"authorNameInfo[0].nameFormat","label":{"en":"Name Format","jp":"フォーマット"},"mask":{},"validation":{'map': ['familyNmAndNm']},"autofill":{'condition': {'either_required': ['familyName', 'firstName']},'value': 'familyNmAndNm'}}, - {"key":"authorNameInfo[0].nameShowFlg","label":{"en":"Name Display","jp":"姓名・言語 表示／非表示"},"mask":{'true': 'Y','false': 'N'},"validation":{'map': ['Y', 'N']},"autofill":""}, - {"key":"authorIdInfo[0].idType","label":{"en":"Identifier Scheme","jp":"外部著者ID 識別子"},"mask":{},"validation":{'validator': {'class_name': 'weko_authors.contrib.validation','func_name': 'validate_identifier_scheme'},'required': {'if': ['authorId']}},"autofill":""}, - {"key":"authorIdInfo[0].authorId","label":{"en":"Identifier","jp":"外部著者ID"},"mask":{},"validation":{'required': {'if': ['idType']}},"autofill":""}, - {"key":"authorIdInfo[0].authorIdShowFlg","label":{"en":"Identifier Display","jp":"外部著者ID 表示／非表示"},"mask":{'true': 'Y','false': 'N'},"validation":{'map': ['Y', 'N']},"autofill":""}, - {"key":"emailInfo[0].email","label":{"en":"Mail Address","jp":"メールアドレス"},"mask":{},"validation":{},"autofill":""}, - {"key":"is_deleted","label":{"en":"Delete Flag","jp":"削除フラグ"},"mask":{'true': 'D','false': ''},"validation":{'map': ['D']},"autofill":""}, + mapping = [ + {"key": "pk_id", "validation": {}}, + {"key": "authorNameInfo[0].familyName", "validation": {}}, + {"key": "authorNameInfo[0].firstName", "validation": {}}, + {"key": "authorNameInfo[0].language", "validation": {"map": ["ja", "en"]}}, + {"key": "authorNameInfo[0].nameFormat", "validation": {"map": ["familyNmAndNm"]}}, + {"key": "authorNameInfo[0].nameShowFlg", "validation": {"map": ["Y", "N"]}}, + { + "key": "authorIdInfo[0].idType", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_identifier_scheme", + } + }, + }, + { + "key": "authorIdInfo[0].authorId", + "validation": {"required": {"if": ["authorIdInfo[0].idType"]}}, + }, + {"key": "authorIdInfo[0].authorIdShowFlg", "validation": {"map": ["Y", "N"]}}, + {"key": "emailInfo[0].email", "validation": {}}, + {"key": "is_deleted", "validation": {"map": ["D"]}}, ] + list_import_id = [] + test = [ + { + "pk_id": "1", + "weko_id": "1", + "current_weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [{"idType": "2", "authorId": "1234", "authorIdShowFlg": "Y"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "update", + } + ] + mocker.patch("weko_authors.utils.WekoAuthors.get_weko_id_by_pk_id",return_value="1") + mocker.patch("weko_authors.utils.check_weko_id_is_exits_for_import",return_value=[]) + result = validate_import_data(file_format,file_data,mapping_ids,mapping,list_import_id) + assert result == test + # pk_id is already in the list_import_id + file_format = "tsv" + file_data = [ + { + "pk_id": "1", + "weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [ + {"idType": "ORCID", "authorId": "1234", "authorIdShowFlg": "Y"} + ], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + }, + { + "pk_id": "1", + "weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [ + {"idType": "ORCID", "authorId": "1234", "authorIdShowFlg": "Y"} + ], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + }, + ] + list_import_id = [] + mocker.patch("weko_authors.utils.WekoAuthors.get_weko_id_by_pk_id",return_value="1") + mocker.patch("weko_authors.utils.check_weko_id_is_exits_for_import",return_value=[]) test = [ - {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'true'}], 'authorIdInfo': [{'idType': 2, 'authorId': '1234', 'authorIdShowFlg': 'true'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': '', 'status': 'update'}, - {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'language': 'ja', 'nameFormat': 'familyNmAndNm', 'nameShowFlg': 'Y'}], 'authorIdInfo': [{'idType': 2, 'authorId': '1234', 'authorIdShowFlg': 'Y'}], 'emailInfo': [{'email': 'test.taro@test.org'}], 'is_deleted': '', 'status': 'update', 'errors': ['There is duplicated data in the tsv file.']} + { + "pk_id": "1", + "weko_id": "1", + "current_weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [{"idType": "2", "authorId": "1234", "authorIdShowFlg": "Y"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "update", + }, + { + "pk_id": "1", + "weko_id": "1", + "current_weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [{"idType": "2", "authorId": "1234", "authorIdShowFlg": "Y"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "update", + "errors": ["There is duplicated data in the tsv file."], + }, ] - result = validate_import_data(file_format,file_data,mapping_ids,mappings) + result = validate_import_data(file_format,file_data,mapping_ids,mapping,list_import_id) assert result == test + # errors_key is ture（check required） + file_format = "tsv" file_data = [ - {'pk_id': None, 'authorIdInfo': [{'idType': 'ORCID', 'authorId': '1234', 'authorIdShowFlg': 'Y'}], 'is_deleted': ''}, - ] - mappings = [ - {"key":"pk_id","label":{"en":"WEKO ID","jp":"WEKO ID"},"mask":{},"validation":{},"autofill":""}, - {"key":"authorIdInfo[0].idType","label":{"en":"Identifier Scheme","jp":"外部著者ID 識別子"},"mask":{},"validation":{'validator': {'class_name': 'weko_authors.contrib.validation','func_name': 'validate_identifier_scheme'}},"autofill":""}, - {"key":"authorIdInfo[0].authorId","label":{"en":"Identifier","jp":"外部著者ID"},"mask":{},"validation":{'required': {'if': ['idType']}},"autofill":""}, - {"key":"authorIdInfo[0].authorIdShowFlg","label":{"en":"Identifier Display","jp":"外部著者ID 表示／非表示"},"mask":{'true': 'Y','false': 'N'},"validation":{'map': ['Y', 'N']},"autofill":""}, - {"key":"is_deleted","label":{"en":"Delete Flag","jp":"削除フラグ"},"mask":{'true': 'D','false': ''},"validation":{'map': ['D']},"autofill":""}, + { + "pk_id": "1", + "weko_id": "1", + "authorNameInfo": [ + { + "familyName": "", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [ + {"idType": "ORCID", "authorId": "1234", "authorIdShowFlg": "Y"} + ], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + } + ] + mapping = [ + {"key": "pk_id", "validation": {}}, + { + "key": "authorNameInfo[0].familyName", + "validation": {"required": {"if": ["firstName"]}}, + }, + {"key": "authorNameInfo[0].firstName", "validation": {}}, + {"key": "authorNameInfo[0].language", "validation": {"map": ["ja", "en"]}}, + { + "key": "authorNameInfo[0].nameFormat", + "validation": {"map": ["familyNmAndNm"]}, + }, + {"key": "authorNameInfo[0].nameShowFlg", "validation": {"map": ["Y", "N"]}}, + { + "key": "authorIdInfo[0].idType", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_identifier_scheme", + } + }, + }, + { + "key": "authorIdInfo[0].authorId", + "validation": {"required": {"if": ["idType"]}}, + }, + {"key": "authorIdInfo[0].authorIdShowFlg", "validation": {"map": ["Y", "N"]}}, + {"key": "emailInfo[0].email", "validation": {}}, + {"key": "is_deleted", "validation": {"map": ["D"]}}, + ] + list_import_id = [] + test = [ + { + "pk_id": "1", + "weko_id": "1", + "current_weko_id": "1", + "authorNameInfo": [ + { + "familyName": "", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [{"idType": "2", "authorId": "1234", "authorIdShowFlg": "Y"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "update", + "errors": ["authorNameInfo[0].familyName is required item."], + } + ] + mocker.patch("weko_authors.utils.WekoAuthors.get_weko_id_by_pk_id",return_value="1") + mocker.patch("weko_authors.utils.check_weko_id_is_exits_for_import",return_value=[]) + result = validate_import_data(file_format,file_data,mapping_ids,mapping,list_import_id) + assert result == test + + # errors_key is ture（check allow data） + file_format = "tsv" + file_data = [ + { + "pk_id": "1", + "weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "de", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [ + {"idType": "ORCID", "authorId": "1234", "authorIdShowFlg": "Y"} + ], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + } + ] + list_import_id = [] + mapping = [ + {"key": "pk_id", "validation": {}}, + {"key": "authorNameInfo[0].familyName", "validation": {}}, + {"key": "authorNameInfo[0].firstName", "validation": {}}, + {"key": "authorNameInfo[0].language", "validation": {"map": ["ja", "en"]}}, + { + "key": "authorNameInfo[0].nameFormat", + "validation": {"map": ["familyNmAndNm"]}, + }, + {"key": "authorNameInfo[0].nameShowFlg", "validation": {"map": ["Y", "N"]}}, + { + "key": "authorIdInfo[0].idType", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_identifier_scheme", + } + }, + }, + { + "key": "authorIdInfo[0].authorId", + "validation": {"required": {"if": ["idType"]}}, + }, + {"key": "authorIdInfo[0].authorIdShowFlg", "validation": {"map": ["Y", "N"]}}, + {"key": "emailInfo[0].email", "validation": {}}, + {"key": "is_deleted", "validation": {"map": ["D"]}}, + ] + test = [ + { + "pk_id": "1", + "weko_id": "1", + "current_weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "de", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [{"idType": "2", "authorId": "1234", "authorIdShowFlg": "Y"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "update", + "errors": ["authorNameInfo[0].language should be set by one of ['ja', 'en']."], + } + ] + mocker.patch("weko_authors.utils.WekoAuthors.get_weko_id_by_pk_id",return_value="1") + mocker.patch("weko_authors.utils.check_weko_id_is_exits_for_import",return_value=[]) + result = validate_import_data(file_format,file_data,mapping_ids,mapping,list_import_id) + assert result == test + + # pk_id is None and errors_msg is exists + file_format = "tsv" + file_data = [ + { + "weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [ + {"idType": "ORCID", "authorId": "1234", "authorIdShowFlg": "Y"} + ], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + } ] mapping_ids = ["pk_id", - "authorIdInfo[0].idType", - "authorIdInfo[0].authorId", - "authorIdInfo[0].authorIdShowFlg", - "is_deleted" - ] - mocker.patch("weko_authors.utils.validate_required",return_value=["required item"]) - mocker.patch("weko_authors.utils.validate_map",return_value=["map item"]) + "authorNameInfo[0].familyName", + "authorNameInfo[0].firstName", + "authorNameInfo[0].language", + "authorNameInfo[0].nameFormat", + "authorNameInfo[0].nameShowFlg", + "authorIdInfo[0].idType", + "authorIdInfo[0].authorId", + "authorIdInfo[0].authorIdShowFlg", + "emailInfo[0].email", + "is_deleted", + "affiliationInfo[0].identifierInfo[0].affiliationIdType" + ] + mapping = [ + {"key": "pk_id", "validation": {}}, + {"key": "authorNameInfo[0].familyName", "validation": {}}, + {"key": "authorNameInfo[0].firstName", "validation": {}}, + {"key": "authorNameInfo[0].language", "validation": {"map": ["ja", "en"]}}, + { + "key": "authorNameInfo[0].nameFormat", + "validation": {"map": ["familyNmAndNm"]}, + }, + {"key": "authorNameInfo[0].nameShowFlg", "validation": {"map": ["Y", "N"]}}, + { + "key": "authorIdInfo[0].idType", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_identifier_scheme", + } + }, + }, + { + "key": "authorIdInfo[0].authorId", + "validation": {"required": {"if": ["idType"]}}, + }, + {"key": "authorIdInfo[0].authorIdShowFlg", "validation": {"map": ["Y", "N"]}}, + {"key": "emailInfo[0].email", "validation": {}}, + {"key": "is_deleted", "validation": {"map": ["D"]}}, + {"key": "affiliationInfo[0].identifierInfo[0].affiliationIdType", "validation": {}} + ] + list_import_id = [] + test = [ + { + "weko_id": "1", + "current_weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [{"idType": "2", "authorId": "1234", "authorIdShowFlg": "Y"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "new", + "errors": ["Specified WEKO ID already exist.", "validator error"], + "warnings": ["idType warning"], + } + ] + mocker.patch("weko_authors.utils.WekoAuthors.get_weko_id_by_pk_id",return_value="1") + mocker.patch("weko_authors.utils.check_weko_id_is_exits_for_import",return_value=["Specified WEKO ID already exist."]) mocker.patch("weko_authors.utils.validate_by_extend_validator",return_value=["validator error"]) mocker.patch("weko_authors.utils.validate_external_author_identifier",return_value="idType warning") - test = [{'pk_id': None, 'authorIdInfo': [{'idType': 2, 'authorId': '1234', 'authorIdShowFlg': 'Y'}], 'is_deleted': '', 'status': 'new', 'errors': ['validator error', 'required item is required item.', "map item should be set by one of ['Y', 'N'].", "map item should be set by one of ['D']."], 'warnings': ['idType warning']}] - result = validate_import_data(file_format,file_data,mapping_ids,mappings) + result = validate_import_data(file_format,file_data,mapping_ids,mapping,list_import_id) + assert result == test + + # autofill and mask is exists + file_format = "tsv" + file_data = [ + { + "pk_id": "1", + "weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [ + {"idType": "ORCID", "authorId": "1234", "authorIdShowFlg": "Y"} + ], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + } + ] + mapping_ids = ["pk_id", + "authorNameInfo[0].familyName", + "authorNameInfo[0].firstName", + "authorNameInfo[0].language", + "authorNameInfo[0].nameFormat", + "authorNameInfo[0].nameShowFlg", + "authorIdInfo[0].idType", + "authorIdInfo[0].authorId", + "authorIdInfo[0].authorIdShowFlg", + "emailInfo[0].email", + "is_deleted", + "mask", + "test", + ] + mapping = [ + {"key": "pk_id", "validation": {}}, + {"key": "authorNameInfo[0].familyName", "validation": {}}, + {"key": "authorNameInfo[0].firstName", "validation": {}}, + {"key": "authorNameInfo[0].language", "validation": {"map": ["ja", "en"]}}, + {"key": "authorNameInfo[0].nameFormat", "validation": {"map": ["familyNmAndNm"]}}, + {"key": "authorNameInfo[0].nameShowFlg", "validation": {"map": ["Y", "N"]}}, + { + "key": "authorIdInfo[0].idType", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_identifier_scheme", + } + }, + }, + { + "key": "authorIdInfo[0].authorId", + "validation": {"required": {"if": ["authorIdInfo[0].idType"]}}, + }, + {"key": "authorIdInfo[0].authorIdShowFlg", "validation": {"map": ["Y", "N"]}}, + {"key": "emailInfo[0].email", "validation": {}}, + {"key": "is_deleted", "validation": {"map": ["D"]}}, + {"autofill": "test","key": "test", "validation": {}}, + {"mask": "test", "key": "test", "validation": {}} + ] + list_import_id = [] + test = [ + { + "pk_id": "1", + "weko_id": "1", + "current_weko_id": "1", + "authorNameInfo": [ + { + "familyName": "テスト", + "firstName": "太郎", + "language": "ja", + "nameFormat": "familyNmAndNm", + "nameShowFlg": "Y", + } + ], + "authorIdInfo": [{"idType": "2", "authorId": "1234", "authorIdShowFlg": "Y"}], + "emailInfo": [{"email": "test.taro@test.org"}], + "is_deleted": "", + "status": "update", + } + ] + mocker.patch("weko_authors.utils.WekoAuthors.get_weko_id_by_pk_id",return_value="1") + mocker.patch("weko_authors.utils.check_weko_id_is_exits_for_import",return_value=[]) + mocker.patch("weko_authors.utils.validate_by_extend_validator",return_value=[]) + mocker.patch("weko_authors.utils.validate_external_author_identifier",return_value="") + mocker.patch("weko_authors.utils.autofill_data") + mocker.patch("weko_authors.utils.convert_data_by_mask") + result = validate_import_data(file_format,file_data,mapping_ids,mapping,list_import_id) assert result == test - # def get_values_by_mapping(keys, data, parent_key=None): # .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_get_values_by_mapping -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -498,7 +1149,7 @@ def test_convert_scheme_to_id(): # def set_record_status(file_format, list_existed_author_id, item, errors, warnings): -# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_set_record_status -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_set_record_status -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_set_record_status(): file_format = "tsv" existed_authors_id = {"1":True} @@ -529,7 +1180,7 @@ def test_set_record_status(): errors = [] warnings = [] set_record_status(file_format,existed_authors_id,item,errors,warnings) - assert errors == ["Specified WEKO ID does not exist."] + assert errors == ["Specified Author ID does not exist."] assert warnings == [] # is_deleted, existed_authors_id.pk_id is None @@ -538,7 +1189,7 @@ def test_set_record_status(): warnings = [] set_record_status(file_format,existed_authors_id,item,errors,warnings) assert item["status"] == "deleted" - assert errors == ["Specified WEKO ID does not exist."] + assert errors == ["Specified Author ID does not exist."] assert warnings == [] existed_authors_id = {"1":True} @@ -559,12 +1210,13 @@ def test_set_record_status(): assert item["status"] == "deleted" # def flatten_authors_mapping(mapping, parent_key=None): -# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_flatten_authors_mapping -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_flatten_authors_mapping -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_flatten_authors_mapping(): data = WEKO_AUTHORS_FILE_MAPPING test_all=[ - {"key":"pk_id","label":{"en":"WEKO ID","jp":"WEKO ID"},"mask":{},"validation":{},"autofill":""}, + {'key': 'pk_id', 'label': {'en': 'Author ID', 'jp': '著者ID'}, 'mask': {}, 'validation': {}, 'autofill': ''}, + {'key': 'weko_id', 'label': {'en': 'WEKO ID', 'jp': 'WEKO ID'}, 'mask': {}, 'validation': {'validator': {'class_name': 'weko_authors.contrib.validation', 'func_name': 'validate_digits_for_wekoid'}}, 'autofill': ''}, {"key":"authorNameInfo[0].familyName","label":{"en":"Family Name","jp":"姓"},"mask":{},"validation":{},"autofill":""}, {"key":"authorNameInfo[0].firstName","label":{"en":"Given Name","jp":"名"},"mask":{},"validation":{},"autofill":""}, {"key":"authorNameInfo[0].language","label":{"en":"Language","jp":"言語"},"mask":{},"validation":{'map': ['ja', 'ja-Kana', 'en', 'fr','it', 'de', 'es', 'zh-cn', 'zh-tw','ru', 'la', 'ms', 'eo', 'ar', 'el', 'ko']},"autofill":""}, @@ -578,6 +1230,7 @@ def test_flatten_authors_mapping(): ] test_keys = ["pk_id", + 'weko_id', "authorNameInfo[0].familyName", "authorNameInfo[0].firstName", "authorNameInfo[0].language", @@ -590,63 +1243,141 @@ def test_flatten_authors_mapping(): "is_deleted" ] all,keys = flatten_authors_mapping(data) + print(all) + print(keys) assert all == test_all assert keys == test_keys # def import_author_to_system(author): -# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_import_author_to_system -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp -def test_import_author_to_system(app,mocker): - # author is None - result = import_author_to_system(None) - assert result == None - - author = { - "status":"", - "is_deleted":"", - } - - # authorIdInfo is None,emailInfo is None,status is new - mock_create = mocker.patch("weko_authors.utils.WekoAuthors.create") - author = { - "status":"new", - "is_deleted":True, +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_import_author_to_system -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_import_author_to_system(app, mocker): + + + author = {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎'}]} + status = 'new' + weko_id = '1234' + force_change_mode = False + test = { + 'pk_id': '1', + 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'fullName': 'テスト 太郎'}], + 'is_deleted': False, + 'authorIdInfo': [], + 'emailInfo': [] } - test = {"is_deleted":True,"authorIdInfo":[],"emailInfo":[]} - result = import_author_to_system(author) - mock_create.assert_called_with(test) - - mocker.patch("weko_authors.utils.get_count_item_link",return_value=1) - # status is not new - author = { - "status":"update", - "is_deleted":True, - "pk_id":"1", - "authorIdInfo": [{ "authorId": "1", "authorIdShowFlg": "true", "idType": "1" }], - "emailInfo": [{ "email": "test.taro@test.org" }] + with patch('weko_authors.utils.check_weko_id_is_exists') as mock_check_weko_id, \ + patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.db.session') as mock_session: + + mock_check_weko_id.return_value = False + import_author_to_system(author, status, weko_id, force_change_mode) + mock_check_weko_id.assert_called_once_with(weko_id, '1') + mock_weko_authors.create.assert_called_once() + actual_author = mock_weko_authors.create.call_args[0][0] + print(actual_author) + + assert actual_author == {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'fullName': 'テスト 太郎'}], 'is_deleted': False, 'authorIdInfo': [{'idType': '1', 'authorId': '1234', 'authorIdShowFlg': 'true'}], 'emailInfo': []} + mock_session.commit.assert_called_once() + + author = {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎'}]} + status = 'update' + weko_id = '1234' + force_change_mode = False + test = { + 'pk_id': '1', + 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'fullName': 'テスト 太郎'}], + 'is_deleted': False, + 'authorIdInfo': [ + { + "idType": "1", + "authorId": "1234", + "authorIdShowFlg": "true" + } + ], + 'emailInfo': [] } + with patch('weko_authors.utils.check_weko_id_is_exists') as mock_check_weko_id, \ + patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.db.session') as mock_session: + + mock_check_weko_id.return_value = False + import_author_to_system(author, status, weko_id, force_change_mode) + mock_check_weko_id.assert_called_once_with(weko_id, '1') + mock_weko_authors.update.assert_called_once() + update_args = mock_weko_authors.update.call_args + actual_author = update_args[0][1] + + assert actual_author == test + mock_session.commit.assert_called_once() + + author = {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎'}]} + status = 'deleted' + weko_id = '1234' + force_change_mode = False test = { - "is_deleted":True, - "pk_id":"1", - "authorIdInfo": [ - {"idType": "1","authorId":"1","authorIdShowFlg": "true"}, - { "authorId": "1", "authorIdShowFlg": "true", "idType": "1" }, - ], - "emailInfo": [{ "email": "test.taro@test.org" }], + 'pk_id': '1', + 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎', 'fullName': 'テスト 太郎'}], + 'is_deleted': False, + 'authorIdInfo': [ + { + "idType": "1", + "authorId": "1234", + "authorIdShowFlg": "true" + } + ], + 'emailInfo': [] + } + with patch('weko_authors.utils.check_weko_id_is_exists') as mock_check_weko_id, \ + patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.db.session') as mock_session, \ + patch('weko_authors.utils.get_count_item_link') as mock_get_count_item_link: - } - mock_update = mocker.patch("weko_authors.utils.WekoAuthors.update") - result = import_author_to_system(author) - mock_update.assert_called_with("1",test) + mock_check_weko_id.return_value = False + mock_get_count_item_link.return_value = 0 + import_author_to_system(author, status, weko_id, force_change_mode) + mock_check_weko_id.assert_called_once_with(weko_id, '1') + mock_weko_authors.update.assert_called_once() + update_args = mock_weko_authors.update.call_args + actual_author = update_args[0][1] + + assert actual_author == test + mock_session.commit.assert_called_once() - # status is deleted - author = { - "status":"deleted", - "is_deleted":True, - "pk_id":"1" - } - with pytest.raises(Exception): - import_author_to_system(author) + author = {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎'}]} + status = 'new' + weko_id = '1234' + force_change_mode = False + with patch('weko_authors.utils.check_weko_id_is_exists') as mock_check_weko_id: + with pytest.raises(Exception) as ex: + mock_check_weko_id.return_value = True + import_author_to_system(author, status, weko_id, force_change_mode) + assert ex.value.args[0]['error_id'] == "WekoID is duplicated" + assert str(ex.value) == {'error_id': "WekoID is duplicated"} + + author = {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎'}]} + status = 'deleted' + weko_id = '1234' + force_change_mode = False + with patch('weko_authors.utils.check_weko_id_is_exists') as mock_check_weko_id,\ + patch('weko_authors.utils.get_count_item_link') as mock_get_count_item_link: + with pytest.raises(Exception) as ex: + mock_check_weko_id.return_value = False + mock_get_count_item_link.return_value = 1 + import_author_to_system(author, status, weko_id, force_change_mode) + assert ex.value.args[0]['error_id'] == "delete_author_link" + assert str(ex.value) == {'error_id': "delete_author_link"} + + author = {'pk_id': '1', 'authorNameInfo': [{'familyName': 'テスト', 'firstName': '太郎'}]} + status = 'update' + weko_id = '1234' + force_change_mode = False + with patch('weko_authors.utils.check_weko_id_is_exists') as mock_check_weko_id: + with pytest.raises(Exception) as ex: + mock_check_weko_id.return_value = True + import_author_to_system(author, status, weko_id, force_change_mode) + assert ex.value.args[0]['error_id'] == "WekoID is duplicated" + assert str(ex.value) == {'error_id': "WekoID is duplicated"} + # def get_count_item_link(pk_id): # .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_get_count_item_link -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_get_count_item_link(app,mocker): @@ -666,4 +1397,1865 @@ def search(self,index,body): record_indexer.client.return_data={"hits":{"total":10}} result = get_count_item_link(1) - assert result == 10 \ No newline at end of file + assert result == 10 + + +# def count_authors(): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_count_authors -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_count_authors(app2, esindex): + import json + index = app2.config["WEKO_AUTHORS_ES_INDEX_NAME"] + doc_type = "author-v1.0.0" + + def register(i): + with open(f"tests/data/test_authors/author{i:02}.json","r") as f: + esindex.index(index=index, doc_type=doc_type, id=f"{i}", body=json.load(f), refresh="true") + + def delete(i): + esindex.delete(index=index, doc_type=doc_type, id=f"{i}", refresh="true") + + # 3 Register 1 author data + register(1) + assert count_authors()['count'] == 1 + delete(1) + + # 4 Register gather_flg = 1 + register(2) + assert count_authors()['count'] == 0 + delete(2) + + # 5 Register is_deleted = "true" + register(3) + assert count_authors()['count'] == 0 + delete(3) + + # 6 Not register author data + assert count_authors()['count'] == 0 + +from weko_authors.utils import validate_weko_id, check_weko_id_is_exists, check_period_date, delete_export_url,\ + handle_exception, export_prefix,check_file_name, clean_deep, update_data_for_weko_link +from redis.exceptions import RedisError +from sqlalchemy.exc import SQLAlchemyError +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestValidateWekoId -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestValidateWekoId: + # 正常系 + def test_validate_weko_id_valid(self, app): + with patch("weko_authors.utils.check_weko_id_is_exists", return_value=False): + result = validate_weko_id("12345") + assert result == (True, None) + + # 異常系 + def test_validate_weko_id_not_half_digit(self, app): + result = validate_weko_id("abcde") + assert result == (False, "not half digit") + + # 異常系 + def test_validate_weko_id_already_exists(self, app): + with patch("weko_authors.utils.check_weko_id_is_exists", return_value=True): + result = validate_weko_id("12345") + assert result == (False, "already exists") + + # 異常系 + def test_validate_weko_id_exception(self, app): + with patch("weko_authors.utils.check_weko_id_is_exists", side_effect=Exception("Test Exception")): + with pytest.raises(Exception, match="Test Exception"): + validate_weko_id("12345") + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestCheckWekoIdIsExists -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestCheckWekoIdIsExists: + # 正常系: weko_idが存在しない場合 + def test_check_weko_id_is_exists_not_exist(self, app): + with patch('weko_authors.utils.RecordIndexer') as MockIndexer: + mock_indexer = MockIndexer.return_value + mock_indexer.client.search.return_value = {'hits': {'hits': []}} + assert check_weko_id_is_exists("12345") == False + + # 正常系: weko_idが存在するが、pk_idが一致する場合 + def test_check_weko_id_is_exists_exist_same_pk_id(self, app): + with patch('weko_authors.utils.RecordIndexer') as MockIndexer: + mock_indexer = MockIndexer.return_value + mock_indexer.client.search.return_value = { + 'hits': { + 'hits': [ + { + '_source': { + 'pk_id': '1', + 'authorIdInfo': [{'idType': '1', 'authorId': '12345'}] + } + } + ] + } + } + assert check_weko_id_is_exists("12345", pk_id="1") == False + + # 異常系: weko_idが存在する場合 + def test_check_weko_id_is_exists_exist(self, app): + with patch('weko_authors.utils.RecordIndexer') as MockIndexer: + mock_indexer = MockIndexer.return_value + mock_indexer.client.search.return_value = { + 'hits': { + 'hits': [ + { + '_source': { + 'pk_id': '2', + 'authorIdInfo': [{'idType': '1', 'authorId': '12345'}] + } + } + ] + } + } + assert check_weko_id_is_exists("12345") == True + + # 異常系: Elasticsearchクライアントが例外をスローする場合 + def test_check_weko_id_is_exists_exception(self, app): + with patch('weko_authors.utils.RecordIndexer') as MockIndexer: + mock_indexer = MockIndexer.return_value + mock_indexer.client.search.side_effect = Exception("Elasticsearch error") + with pytest.raises(Exception): + check_weko_id_is_exists("12345") + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestCheckPeriodDate -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestCheckPeriodDate: +# 正常系: affiliationInfoが存在し、期間が正しい場合 + def test_check_period_date_valid(self, app): + data = { + "affiliationInfo": [ + { + "affiliationPeriodInfo": [ + {"periodStart": "2020-01-01", "periodEnd": "2021-01-01"} + ] + } + ] + } + assert check_period_date(data) == (True, None) + + # 異常系: periodStartが日付形式でない場合 + def test_check_period_date_invalid_start_date(self, app): + data = { + "affiliationInfo": [ + { + "affiliationPeriodInfo": [ + {"periodStart": "2020-13-01"} + ] + } + ] + } + assert check_period_date(data) == (False, "not date format") + + # 異常系: periodEndが日付形式でない場合 + def test_check_period_date_invalid_end_date(self, app): + data = { + "affiliationInfo": [ + { + "affiliationPeriodInfo": [ + {"periodEnd": "2020-13-01"} + ] + } + ] + } + assert check_period_date(data) == (False, "not date format") + + # 異常系: periodStartがperiodEndより後の場合 + def test_check_period_date_start_after_end(self, app): + data = { + "affiliationInfo": [ + { + "affiliationPeriodInfo": [ + {"periodStart": "2021-01-01", "periodEnd": "2020-01-01"} + ] + } + ] + } + assert check_period_date(data) == (False, "start is after end") + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestDeleteExportUrl -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestDeleteExportUrl: + # 正常系: キャッシュキーが存在する場合 + def test_delete_export_url_key_exists(self, app2): + with patch('weko_authors.utils.current_cache') as mock_cache: + delete_export_url() + mock_cache.delete.assert_called_once_with(current_app.config["WEKO_AUTHORS_EXPORT_CACHE_URL_KEY"]) + + # 正常系: キャッシュキーが存在しない場合 + def test_delete_export_url_key_not_exists(self, app2): + with patch('weko_authors.utils.current_cache') as mock_cache: + mock_cache.delete.return_value = None # Simulate key not existing + delete_export_url() + mock_cache.delete.assert_called_once_with(current_app.config["WEKO_AUTHORS_EXPORT_CACHE_URL_KEY"]) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestHandleException -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestHandleException: +# 正常系: リトライ回数が残っている場合 + def test_handle_exception_retry(self, app2): + with patch('weko_authors.utils.current_app') as mock_app, patch('weko_authors.utils.sleep') as mock_sleep: + mock_app.logger = MagicMock() + handle_exception(Exception("Test error"), attempt=0, retrys=3, interval=1, stop_point=0) + mock_sleep.assert_called_once_with(1) + + # 異常系: リトライ回数が残っていない場合 + def test_handle_exception_no_retry(self, app2): + with patch('weko_authors.utils.current_app') as mock_app: + mock_app.logger = MagicMock() + with pytest.raises(Exception): + handle_exception(Exception("Test error"), attempt=2, retrys=3, interval=1, stop_point=0) + + # 異常系: リトライ回数が残っていない場合、かつstop_pointが指定されている場合 + def test_handle_exception_no_retry_with_stop_point(self, app2): + with patch('weko_authors.utils.current_app') as mock_app, patch('weko_authors.utils.update_cache_data') as mock_update_cache: + mock_app.logger = MagicMock() + with pytest.raises(Exception): + handle_exception(Exception("Test error"), attempt=2, retrys=3, interval=1, stop_point=5) + mock_update_cache.assert_called_once_with( + mock_app.config["WEKO_AUTHORS_EXPORT_CACHE_STOP_POINT_KEY"], + 5, + mock_app.config["WEKO_AUTHORS_CACHE_TTL"] + ) + +# @.tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestExportAuthors -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestExportAuthors: + + @pytest.fixture + def setup_app_config(self, app, db): + """テスト用のapp configを設定するフィクスチャ""" + app_config = { + "WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY": 3, + "WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL": 1, + "WEKO_AUTHORS_EXPORT_BATCH_SIZE": 1000, + "WEKO_AUTHORS_EXPORT_CACHE_STOP_POINT_KEY": "stop_point_key", + "WEKO_AUTHORS_FILE_MAPPING": {"key": "value"}, + "WEKO_AUTHORS_FILE_MAPPING_FOR_AFFILIATION": {"aff_key": "aff_value"}, + "WEKO_AUTHORS_EXPORT_CACHE_TEMP_FILE_PATH_KEY": "temp_file_path_key", + "WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY": "target_cache_key" + } + return app_config + + @pytest.fixture + def mock_dependencies(self, setup_app_config): + """依存関係をモックするフィクスチャ""" + with patch('builtins.open') as mock_open, \ + patch('weko_authors.utils.current_app') as mock_app, \ + patch('weko_authors.utils.current_cache') as mock_cache, \ + patch('weko_authors.utils.WekoAuthors') as mock_weko, \ + patch('weko_authors.utils.db') as mock_db, \ + patch('weko_authors.utils.os') as mock_os, \ + patch('invenio_files_rest.models.FileInstance') as mock_file, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.get_export_url') as mock_get_url, \ + patch('weko_authors.utils.write_to_tempfile') as mock_write, \ + patch('weko_authors.utils.handle_exception') as mock_handle, \ + patch('weko_authors.utils.io.BufferedReader') as mock_reader, \ + patch('weko_authors.utils.traceback') as mock_traceback, \ + patch('weko_authors.utils.stdout') as mock_stdout: + + # モックオブジェクトの設定 + mock_app.config = setup_app_config + mock_app.logger = MagicMock() + + mock_cache.get.side_effect = lambda key: None if key == "stop_point_key" else "/tmp/test_file.csv" + mock_cache.delete = MagicMock() + mock_cache.set = MagicMock() + + mock_weko.get_records_count.return_value = 2000 + mock_weko.mapping_max_item.return_value = ({"key": "value"}, {"aff_key": "aff_value"}) + mock_weko.get_identifier_scheme_info.return_value = {"scheme1": "info1"} + mock_weko.get_affiliation_identifier_scheme_info.return_value = {"aff_scheme1": "aff_info1"} + mock_weko.get_by_range.return_value = [{"id": 1}, {"id": 2}] + mock_weko.prepare_export_data.return_value = (["header"], ["label_en"], ["label_jp"], [["data1"], ["data2"]]) + + mock_get_url.return_value = None + + mock_file_instance = MagicMock() + mock_file_instance.uri = "file://test_uri" + mock_file.create.return_value = mock_file_instance + mock_file.get_by_uri = MagicMock(return_value=mock_file_instance) + + mock_default_location = MagicMock() + mock_default_location.uri = "default_uri" + mock_location.get_default.return_value = mock_default_location + + mock_reader_instance = MagicMock() + mock_reader.return_value = mock_reader_instance + + mock_db.session.commit = MagicMock() + mock_db.session.rollback = MagicMock() + + mock_os.remove = MagicMock() + + yield { + 'app': mock_app, + 'cache': mock_cache, + 'weko': mock_weko, + 'db': mock_db, + 'os': mock_os, + 'file': mock_file, + 'location': mock_location, + 'get_url': mock_get_url, + 'write': mock_write, + 'handle': mock_handle, + 'reader': mock_reader, + 'reader_instance': mock_reader_instance, + 'file_instance': mock_file_instance, + 'traceback': mock_traceback, + 'stdout': mock_stdout + } + + + def test_normal_case(self, mock_dependencies): + """ + テストケース1: 正常系 - 著者データがある場合 + 条件: 著者データが存在し、すべての処理が正常に完了する + 入力: なし（関数内部でデータを取得） + 期待結果: + - 非nullのfile_uriが返される + - 一時ファイルが作成され、その後削除される + - キャッシュに"author_db"が設定される + - 著者データが正しくエクスポートされる + """ + + + result = export_authors() + + # 検証 + assert result == "file://test_uri" + mock_dependencies['cache'].delete.assert_called_once_with("stop_point_key") + mock_dependencies['write'].assert_called() + mock_dependencies['os'].remove.assert_called_once_with("/tmp/test_file.csv") + mock_dependencies['db'].session.commit.assert_called_once() + mock_dependencies['cache'].set.assert_called_once_with( + "target_cache_key", "author_db", timeout=0 + ) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestExportAuthors::test_with_stop_point -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_with_stop_point(self, mock_dependencies): + """ + テストケース2: 正常系 - stop_pointが設定されている場合 + 条件: キャッシュにstop_pointが設定されている + 入力: stop_point = 500（キャッシュ内） + 期待結果: + - start_pointが500から開始される + - 処理が正常に完了し、非nullのfile_uriが返される + - キャッシュからstop_pointが削除される + """ + # stop_pointを設定 + mock_dependencies['cache'].get.side_effect = lambda key: 500 if key == "stop_point_key" else "/tmp/test_file.csv" + + result = export_authors() + + # 検証 + assert result == "file://test_uri" + mock_dependencies['cache'].delete.assert_called_once_with("stop_point_key") + # 500から始まって1500で終わることを確認 + mock_dependencies['app'].logger.info.assert_any_call("Export authors start_point：500") + mock_dependencies['weko'].get_by_range.assert_called_with(1500, 1000, False, False) + + + def test_with_existing_cache_url(self, mock_dependencies): + """ + テストケース3: 正常系 - キャッシュURLが存在する場合 + 条件: get_export_url()が既存のキャッシュURLを返す + 入力: cache_url = {'file_uri': 'existing_uri'} + 期待結果: + - 既存のファイルが更新される + - 同じfile_uriが返される + """ + # 既存のキャッシュURLを設定 + mock_dependencies['get_url'].return_value = {'file_uri': 'existing_uri'} + + + + result = export_authors() + + # 検証 + assert result == "file://test_uri" + mock_dependencies['file'].get_by_uri.assert_called_once_with('existing_uri') + mock_dependencies['file_instance'].set_contents.assert_called_once_with(mock_dependencies['reader_instance']) + mock_dependencies['file'].create.assert_not_called() + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestExportAuthors::test_sqlalchemy_error_on_mapping_with_retry_success -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_sqlalchemy_error_on_mapping_with_retry_success(self, mock_dependencies): + """ + テストケース4: 異常系 - マッピング取得時のSQLAlchemyエラー（リトライ成功） + 条件: 最初のSQLAlchemyエラーが発生し、リトライが成功する + 入力: 1回目のマッピング取得時にSQLAlchemyErrorを発生させ、2回目は成功 + 期待結果: + - リトライ後に処理が続行され、正常に完了する + - 非nullのfile_uriが返される + """ + # 1回目のマッピング取得でエラー、2回目は成功するように設定 + mock_dependencies['weko'].get_records_count.side_effect = [SQLAlchemyError("DB Error"), 2000] + + + + result = export_authors() + + # 検証 + assert result == "file://test_uri" + mock_dependencies['handle'].assert_called_once() + mock_dependencies['weko'].get_records_count.assert_called_with(False, False) + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestExportAuthors::test_redis_error_on_mapping_with_retry_success -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_redis_error_on_mapping_with_retry_success(self, mock_dependencies): + """ + テストケース5: 異常系 - マッピング取得時のRedisエラー（リトライ成功） + 条件: 最初のRedisエラーが発生し、リトライが成功する + 入力: 1回目のマッピング取得時にRedisErrorを発生させ、2回目は成功 + 期待結果: + - リトライ後に処理が続行され、正常に完了する + - 非nullのfile_uriが返される + """ + # 1回目のマッピング取得でRedisエラー、2回目は成功するように設定 + mock_dependencies['weko'].mapping_max_item.side_effect = [RedisError("Redis Error"), ({"key": "value"}, {"aff_key": "aff_value"})] + + + + result = export_authors() + + # 検証 + assert result == "file://test_uri" + mock_dependencies['handle'].assert_called_once() + + + def test_timeout_error_on_mapping_with_retry_success(self, mock_dependencies): + """ + テストケース6: 異常系 - マッピング取得時のTimeoutエラー（リトライ成功） + 条件: 最初のTimeoutエラーが発生し、リトライが成功する + 入力: 1回目のマッピング取得時にTimeoutErrorを発生させ、2回目は成功 + 期待結果: + - リトライ後に処理が続行され、正常に完了する + - 非nullのfile_uriが返される + """ + # 1回目のマッピング取得でTimeoutエラー、2回目は成功するように設定 + mock_dependencies['weko'].get_identifier_scheme_info.side_effect = [TimeoutError("Timeout"), {"scheme1": "info1"}] + + + + result = export_authors() + + # 検証 + assert result == "file://test_uri" + mock_dependencies['handle'].assert_called_once() + + + def test_sqlalchemy_error_on_author_info_with_retry_success(self, mock_dependencies): + """ + テストケース7: 異常系 - 著者情報取得時のSQLAlchemyエラー（リトライ成功） + 条件: 著者情報取得時にSQLAlchemyErrorが発生し、リトライが成功する + 入力: 1回目の著者情報取得時にSQLAlchemyErrorを発生させ、2回目は成功 + 期待結果: + - リトライ後に処理が続行され、正常に完了する + - 非nullのfile_uriが返される + """ + # 著者情報取得でSQLAlchemyエラー、2回目は成功するように設定 + mock_dependencies['weko'].get_by_range.side_effect = [SQLAlchemyError("DB Error"), [{"id": 1}, {"id": 2}]] + + + + result = export_authors() + + # 検証 + mock_dependencies['handle'].assert_called_once_with( + mock_dependencies['handle'].call_args[0][0], + 0, + 3, + 1, + stop_point=0 + ) + + + def test_redis_error_on_author_info_with_retry_success(self, mock_dependencies): + """ + テストケース8: 異常系 - 著者情報取得時のRedisエラー（リトライ成功） + 条件: 著者情報取得時にRedisErrorが発生し、リトライが成功する + 入力: 1回目の著者情報取得時にRedisErrorを発生させ、2回目は成功 + 期待結果: + - リトライ後に処理が続行され、正常に完了する + - 非nullのfile_uriが返される + """ + # 著者情報取得でRedisエラー、2回目は成功するように設定 + mock_dependencies['weko'].prepare_export_data.side_effect = [ + RedisError("Redis Error"), + (["header"], ["label_en"], ["label_jp"], [["data1"], ["data2"]]) + ] + + + + result = export_authors() + + # 検証 + mock_dependencies['handle'].assert_called_once_with( + mock_dependencies['handle'].call_args[0][0], + 0, + 3, + 1, + stop_point=0 + ) + + # .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestExportAuthors::test_timeout_error_on_author_info_with_retry_success -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_timeout_error_on_author_info_with_retry_success(self, mock_dependencies): + """ + テストケース9: 異常系 - 著者情報取得時のTimeoutエラー（リトライ成功） + 条件: 著者情報取得時にTimeoutErrorが発生し、リトライが成功する + 入力: 1回目の著者情報取得時にTimeoutErrorを発生させ、2回目は成功 + 期待結果: + - リトライ後に処理が続行され、正常に完了する + """ + # 1回目のマッピング取得でRedisエラー、2回目は成功するように設定 + mock_dependencies['weko'].prepare_export_data.side_effect = [TimeoutError("TimeoutError"), ({"key": "value"}, {"aff_key": "aff_value"})] + + result = export_authors() + + # 検証 + mock_dependencies['handle'].assert_called_once() + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestExportPrefix -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestExportPrefix: + def test_export_prefix_id_prefix_success(self, app): + """正常系: id_prefixの正常エクスポート""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db: + + # モックの設定 + mock_weko_authors.get_id_prefix_all.return_value = ['test_prefix'] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'test_uri' + mock_file_instance.create.return_value = mock_file + + mock_location_instance = Mock() + mock_location_instance.uri = 'default_uri' + mock_location.get_default.return_value = mock_location_instance + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + app.config['WEKO_ADMIN_OUTPUT_FORMAT'] = 'tsv' + app.config['WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY'] = 'test_cache_key' + + result = export_prefix('id_prefix') + + # 検証 + assert result == 'test_uri' + mock_weko_authors.get_id_prefix_all.assert_called_once() + mock_weko_authors.prepare_export_prefix.assert_called_once_with('id_prefix', ['test_prefix']) + mock_file.set_contents.assert_called_once() + mock_db.session.commit.assert_called_once() + mock_current_cache.set.assert_called_once_with('test_cache_key', 'id_prefix', timeout=0) + + def test_export_prefix_affiliation_id_success(self, app): + """正常系: affiliation_idの正常エクスポート""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db: + + # モックの設定 + mock_weko_authors.get_affiliation_id_all.return_value = ['test_affiliation'] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'test_uri' + mock_file_instance.create.return_value = mock_file + + mock_location_instance = Mock() + mock_location_instance.uri = 'default_uri' + mock_location.get_default.return_value = mock_location_instance + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + app.config['WEKO_ADMIN_OUTPUT_FORMAT'] = 'tsv' + app.config['WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY'] = 'test_cache_key' + + result = export_prefix('affiliation_id') + + # 検証 + assert result == 'test_uri' + mock_weko_authors.get_affiliation_id_all.assert_called_once() + mock_weko_authors.prepare_export_prefix.assert_called_once_with('affiliation_id', ['test_affiliation']) + mock_file.set_contents.assert_called_once() + mock_db.session.commit.assert_called_once() + mock_current_cache.set.assert_called_once_with('test_cache_key', 'affiliation_id', timeout=0) + + def test_export_prefix_csv_format(self, app): + """正常系: CSVフォーマットでの出力""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db: + + # モックの設定 + mock_weko_authors.get_id_prefix_all.return_value = ['test_prefix'] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'test_uri' + mock_file_instance.create.return_value = mock_file + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + app.config['WEKO_ADMIN_OUTPUT_FORMAT'] = 'csv' + + result = export_prefix('id_prefix') + + # 検証 + assert result == 'test_uri' + # CSVフォーマットで書き込まれたことを検証（実装依存のため詳細な検証は省略） + + def test_export_prefix_tsv_format(self, app): + """正常系: TSVフォーマットでの出力（デフォルト）""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db: + + # モックの設定 + mock_weko_authors.get_id_prefix_all.return_value = ['test_prefix'] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'test_uri' + mock_file_instance.create.return_value = mock_file + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + app.config['WEKO_ADMIN_OUTPUT_FORMAT'] = 'tsv' + + result = export_prefix('id_prefix') + + # 検証 + assert result == 'test_uri' + # TSVフォーマットで書き込まれたことを検証（実装依存のため詳細な検証は省略） + + def test_export_prefix_with_cache_url(self, app): + """正常系: キャッシュURLが存在する場合""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db: + + # モックの設定 + mock_weko_authors.get_id_prefix_all.return_value = ['test_prefix'] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = {'file_uri': 'cached_uri'} + + mock_file = Mock() + mock_file.uri = 'cached_uri' + mock_file_instance.get_by_uri.return_value = mock_file + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + app.config['WEKO_ADMIN_OUTPUT_FORMAT'] = 'tsv' + app.config['WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY'] = 'test_cache_key' + + result = export_prefix('id_prefix') + + # 検証 + assert result == 'cached_uri' + mock_file_instance.get_by_uri.assert_called_once_with('cached_uri') + assert mock_file.writable == True + mock_file.set_contents.assert_called_once() + mock_db.session.commit.assert_called_once() + + def test_export_prefix_without_cache_url(self, app): + """正常系: キャッシュURLが存在しない場合""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db: + + # モックの設定 + mock_weko_authors.get_id_prefix_all.return_value = ['test_prefix'] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'new_uri' + mock_file_instance.create.return_value = mock_file + + mock_location_instance = Mock() + mock_location_instance.uri = 'default_uri' + mock_location.get_default.return_value = mock_location_instance + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + + result = export_prefix('id_prefix') + + # 検証 + assert result == 'new_uri' + mock_file_instance.create.assert_called_once() + mock_location.get_default.assert_called_once() + + def test_export_prefix_sqlalchemy_error_retry_success(self, app): + """異常系: SQLAlchemyエラー発生（リトライ成功）""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db, \ + patch('weko_authors.utils.handle_exception') as mock_handle_exception: + + # 1回目はエラー、2回目は成功するように設定 + mock_weko_authors.get_id_prefix_all.side_effect = [SQLAlchemyError("DB Error"), ['test_prefix']] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'test_uri' + mock_file_instance.create.return_value = mock_file + + mock_location_instance = Mock() + mock_location_instance.uri = 'default_uri' + mock_location.get_default.return_value = mock_location_instance + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + + result = export_prefix('id_prefix') + + # 検証 + assert result == 'test_uri' + assert mock_weko_authors.get_id_prefix_all.call_count == 2 + mock_handle_exception.assert_called_once() + + def test_export_prefix_sqlalchemy_error_all_fails(self, app): + """異常系: SQLAlchemyエラー発生（全リトライ失敗）""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.handle_exception') as mock_handle_exception: + + # 全てのリトライでエラーを発生させる + mock_weko_authors.get_id_prefix_all.side_effect = SQLAlchemyError("DB Error") + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + + result = export_prefix('id_prefix') + + # 検証 + assert result is None + assert mock_weko_authors.get_id_prefix_all.call_count == 3 + assert mock_handle_exception.call_count == 3 + + def test_export_prefix_redis_error_retry_success(self, app): + """異常系: RedisErrorエラー発生（リトライ成功）""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db, \ + patch('weko_authors.utils.handle_exception') as mock_handle_exception: + + # モックの設定 + mock_weko_authors.get_id_prefix_all.return_value = ['test_prefix'] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'test_uri' + mock_file_instance.create.return_value = mock_file + + mock_location_instance = Mock() + mock_location_instance.uri = 'default_uri' + mock_location.get_default.return_value = mock_location_instance + + # 1回目はエラー、2回目は成功するように設定 + mock_current_cache.set.side_effect = [RedisError("Redis Error"), None] + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + app.config['WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY'] = 'test_cache_key' + + result = export_prefix('id_prefix') + + # 検証 + assert result == 'test_uri' + mock_handle_exception.assert_called_once() + assert mock_current_cache.set.call_count == 2 + + def test_export_prefix_redis_error_all_fails(self, app): + """異常系: RedisErrorエラー発生（全リトライ失敗）""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db, \ + patch('weko_authors.utils.handle_exception') as mock_handle_exception: + + # モックの設定 + mock_weko_authors.get_id_prefix_all.return_value = ['test_prefix'] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'test_uri' + mock_file_instance.create.return_value = mock_file + + mock_location_instance = Mock() + mock_location_instance.uri = 'default_uri' + mock_location.get_default.return_value = mock_location_instance + + # 全てのリトライでエラーを発生させる + mock_current_cache.set.side_effect = RedisError("Redis Error") + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + app.config['WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY'] = 'test_cache_key' + + result = export_prefix('id_prefix') + + # 検証 + assert result is 'test_uri' + assert mock_handle_exception.call_count == 3 + + def test_export_prefix_timeout_error_retry_success(self, app): + """異常系: TimeoutErrorエラー発生（リトライ成功）""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.get_export_url') as mock_get_export_url, \ + patch('invenio_files_rest.models.FileInstance') as mock_file_instance, \ + patch('invenio_files_rest.models.Location') as mock_location, \ + patch('weko_authors.utils.current_cache') as mock_current_cache, \ + patch('weko_authors.utils.db') as mock_db, \ + patch('weko_authors.utils.handle_exception') as mock_handle_exception: + + # 1回目はエラー、2回目は成功するように設定 + mock_weko_authors.get_id_prefix_all.side_effect = [TimeoutError("Timeout"), ['test_prefix']] + mock_weko_authors.prepare_export_prefix.return_value = [['scheme1', 'name1', 'url1', 'false']] + mock_get_export_url.return_value = None + + mock_file = Mock() + mock_file.uri = 'test_uri' + mock_file_instance.create.return_value = mock_file + + mock_location_instance = Mock() + mock_location_instance.uri = 'default_uri' + mock_location.get_default.return_value = mock_location_instance + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + + result = export_prefix('id_prefix') + + # 検証 + assert result == 'test_uri' + assert mock_weko_authors.get_id_prefix_all.call_count == 2 + mock_handle_exception.assert_called_once() + + def test_export_prefix_timeout_error_all_fails(self, app): + """異常系: TimeoutErrorエラー発生（全リトライ失敗）""" + # モックの準備 + with patch('weko_authors.utils.WekoAuthors') as mock_weko_authors, \ + patch('weko_authors.utils.handle_exception') as mock_handle_exception: + + # 全てのリトライでエラーを発生させる + mock_weko_authors.get_id_prefix_all.side_effect = TimeoutError("Timeout") + + # テスト対象の関数を実行 + app.config['WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY'] = 3 + app.config['WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL'] = 0 + + result = export_prefix('id_prefix') + + # 検証 + assert result is None + assert mock_weko_authors.get_id_prefix_all.call_count == 3 + assert mock_handle_exception.call_count == 3 + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestCheckFileName -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestCheckFileName: + def test_check_file_name_author_db(self, app): + """正常系: export_targetがauthor_dbの場合""" + app.config['WEKO_AUTHORS_EXPORT_FILE_NAME'] = 'author_db_export.tsv' + result = check_file_name('author_db') + assert result == 'author_db_export.tsv' + + def test_check_file_name_id_prefix(self, app): + """正常系: export_targetがid_prefixの場合""" + app.config['WEKO_AUTHORS_ID_PREFIX_EXPORT_FILE_NAME'] = 'id_prefix_export.tsv' + result = check_file_name('id_prefix') + assert result == 'id_prefix_export.tsv' + + def test_check_file_name_affiliation_id(self, app): + """正常系: export_targetがaffiliation_idの場合""" + app.config['WEKO_AUTHORS_AFFILIATION_EXPORT_FILE_NAME'] = 'affiliation_id_export.tsv' + result = check_file_name('affiliation_id') + assert result == 'affiliation_id_export.tsv' + + def test_check_file_name_invalid_target(self, app): + """異常系: export_targetが無効な場合""" + result = check_file_name('invalid_target') + assert result == '' + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestCleanDeep -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestCleanDeep: + # 正常系 + # 条件: 辞書の中にNoneや空文字が含まれている + # 入力: {'fullname': 'Jane Doe', 'warnings': None, 'email': {"test": "", "test2": "test2"}, 'test': [{"test": ""}, {"test2": "test2"}]} + # 期待結果: {'fullname': 'Jane Doe', 'email': {"test2": "test2"}, 'test': [{"test2": "test2"}]} + def test_clean_deep_normal_case(self, app): + data = {'fullname': 'Jane Doe', 'warnings': None, 'email': {"test": "", "test2": "test2"}, 'test': [{"test": ""}, {"test2": "test2"}]} + expected = {'fullname': 'Jane Doe', 'email': {"test2": "test2"}, 'test': [{"test2": "test2"}]} + assert clean_deep(data) == expected + +# def update_cache_data(key: str, value: str, timeout=None): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_update_cache_data -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_update_cache_data(app): + with patch('weko_authors.utils.current_cache') as mock_cache: + update_cache_data('test', 'test', 100) + mock_cache.set.assert_called_once_with('test', 'test', timeout=100) + + with patch('weko_authors.utils.current_cache') as mock_cache: + update_cache_data('test', 'test') + mock_cache.set.assert_called_once_with('test', 'test') + +# def write_tmp_part_file(part_num, file_data, temp_file_path): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_write_tmp_part_file -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_write_tmp_part_file(app): + + with patch("weko_authors.utils.open") as mock_writer: + update_cache_data(current_app.config.get("WEKO_AUTHORS_IMPORT_TEMP_FOLDER_PATH"), "/data/test_over_max") + write_tmp_part_file(1, [{"key": "value"}], "temp_file_path") + mock_writer.assert_called() + + +# def unpackage_and_check_import_file_for_prefix(file_format, file_name, temp_file): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_unpackage_and_check_import_file_for_prefix -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_unpackage_and_check_import_file_for_prefix(app): + data = [ + [], + ["scheme", "name", "url", "is_deleted"], + ["123", "aaa", "http://example.com", ''], + ["#456", "bbb", "http://example.org", ''], + ["789", "ccc", "http://example.net", ''], + ["789", "ccc", "http://example.net", ''], + ["789", "ccc", "http://example.net", ''], + ["789", "ccc", "http://example.net", ''], + ["789", "ccc", "http://example.net", ''], + ["789", "ccc", "http://example.net", ''], + ["789", "ccc", "http://example.net", ''], + ] + + with patch("weko_authors.utils.getEncode") as mock_getEncode,\ + patch("weko_authors.utils.open") as mock_writer,\ + patch("weko_authors.utils.csv.reader") as mock_csv: + mock_getEncode.return_value = 'path/to/tempfile' + mock_csv.return_value = iter(data) + result = unpackage_and_check_import_file_for_prefix('csv', 'test.csv', 'path/to/tempfile') + + data_2 = [ + [], + ["scheme", "scheme", "scheme", "scheme"], + ["123", "aaa", "http://example.com", ''], + ["456", "bbb", "http://example.org", ''], + ["789", "ccc", "http://example.net", ''] + ] + + with patch("weko_authors.utils.getEncode") as mock_getEncode,\ + patch("weko_authors.utils.open") as mock_writer,\ + patch("weko_authors.utils.csv.reader") as mock_csv: + with pytest.raises(Exception) as msg: + mock_getEncode.return_value = 'path/to/tempfile' + mock_csv.return_value = iter(data_2) + result = unpackage_and_check_import_file_for_prefix('tsv', 'test.csv', 'path/to/tempfile') + assert str(msg.value) == "{'error_msg': 'The following metadata keys are duplicated.
scheme'}" + + data_3 = [ + [], + ["scheme", "name", "url", "invalid_field"], + ["123", "aaa", "http://example.com"], + ["456", "bbb", "http://example.org"], + ["789", "ccc", "http://example.net"] + ] + + with patch("weko_authors.utils.getEncode") as mock_getEncode,\ + patch("weko_authors.utils.open") as mock_writer,\ + patch("weko_authors.utils.csv.reader") as mock_csv: + with pytest.raises(Exception) as msg: + mock_getEncode.return_value = 'path/to/tempfile' + mock_csv.return_value = iter(data_3) + unpackage_and_check_import_file_for_prefix('csv', 'test.csv', 'path/to/tempfile') + assert str(msg.value) == "{'error_msg': 'Specified item does not consistency with DB item.
invalid_field'}" + + data_4 = [ + [], + ["scheme", "name", "url", "is_deleted"], + ["123", "aaa", "http://example.com", ''], + ["#456", "bbb", "http://example.org", ''], + ["789", "ccc", "http://example.net", ''], + ] + + with patch("weko_authors.utils.getEncode") as mock_getEncode,\ + patch("weko_authors.utils.open") as mock_writer,\ + patch("weko_authors.utils.handle_check_consistence_with_mapping_for_prefix") as mock_check,\ + patch("weko_authors.utils.csv.reader") as mock_csv: + with pytest.raises(UnicodeDecodeError) as msg: + mock_getEncode.return_value = 'path/to/tempfile' + mock_csv.return_value = iter(data_4) + # unicode_error = UnicodeDecodeError('utf-8', b'\x80abc', 0, 1, 'invalid start byte') + # mock_csv.side_effect = unicode_error + mock_check.side_effect = UnicodeDecodeError("utf-8", b'', 0, 1, "invalid start byte") + result = unpackage_and_check_import_file_for_prefix('csv', 'test.csv', 'path/to/tempfile') + assert str(msg.value) == "{'error_msg': 'The following metadata keys are duplicated.
scheme'}" + + data_5 = [ + [], + ["scheme", "name", "url", "is_deleted"], + [123, "aaa", "http://example.com", ''], + ] + + with patch("weko_authors.utils.getEncode") as mock_getEncode,\ + patch("weko_authors.utils.open") as mock_writer,\ + patch("weko_authors.utils.csv.reader") as mock_csv: + with pytest.raises(Exception) as msg: + mock_getEncode.return_value = 'path/to/tempfile' + mock_csv.return_value = iter(data_5) + unpackage_and_check_import_file_for_prefix('csv', 'test.csv', 'path/to/tempfile') + assert str(msg.value) == "{'error_msg': 'The following metadata keys are duplicated.
scheme'}" + + data_6 = [ + [], + ["scheme", "name", "url", "is_deleted"], + ["123", "aaa", "http://example.com", ''], + ["test"], + 'aaaaaaaaaaa' + ] + + with patch("weko_authors.utils.getEncode") as mock_getEncode,\ + patch("weko_authors.utils.open") as mock_writer,\ + patch("weko_authors.utils.csv.reader") as mock_csv: + with pytest.raises(Exception) as msg: + mock_getEncode.return_value = 'path/to/tempfile' + mock_csv.return_value = iter(data_6) + unpackage_and_check_import_file_for_prefix('csv', 'test.csv', 'path/to/tempfile') + assert str(msg.value) == "{'error_msg': 'The following metadata keys are duplicated.
scheme'}" + + +# def get_check_base_name(): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_get_check_base_name -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_get_check_base_name(app): + update_cache_data(current_app.config["WEKO_AUTHORS_IMPORT_CACHE_USER_TSV_FILE_KEY"], "/data/test_over_max") + result = get_check_base_name() + + assert result == "test_over_max-check" + + +# def get_check_result(entry): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_get_check_result -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_get_check_result(app): + result = get_check_result({"errors": ["error1", "error2"], "status": "new"}) + assert result == "Error: error1 error2" + + result = get_check_result({"errors": [], "status": "new"}) + assert result == "Register" + + result = get_check_result({"errors": [], "status": "update"}) + assert result == "Update" + + result = get_check_result({"errors": [], "status": "deleted"}) + assert result == "Delete" + + result = get_check_result({"errors": [], "status": "other"}) + assert result == "other" + + +# def handle_check_consistence_with_mapping_for_prefix(keys, header): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_handle_check_consistence_with_mapping_for_prefix -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_handle_check_consistence_with_mapping_for_prefix(app): + result = handle_check_consistence_with_mapping_for_prefix(["scheme", "name", "url", "is_deleted"], ["scheme", "name", "url", "is_deleted"]) + assert result ==[] + + result = handle_check_consistence_with_mapping_for_prefix(["scheme", "name", "url", "is_deleted"], ["scheme", "name", "url", "is_deleted", "extra_field"]) + assert result == ["extra_field"] + + +# def check_import_data_for_prefix(target, file_name: str, file_content: str): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_check_import_data_for_prefix -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_check_import_data_for_prefix(app): + data = ['aaa', 'bbb'] + + with patch('weko_authors.utils.unpackage_and_check_import_file_for_prefix') as mock_file, \ + patch('weko_authors.utils.validate_import_data_for_prefix') as mock_data: + mock_file.return_value = data + mock_data.return_value = data + result = check_import_data_for_prefix("id_prefix", "test.tsv", "dGVzdA==") + assert len(result['list_import_data']) > 0 + + with patch('weko_authors.utils.unpackage_and_check_import_file_for_prefix') as mock_file, \ + patch('weko_authors.utils.validate_import_data_for_prefix') as mock_data: + mock_file.return_value = data + mock_data.return_value = data + result = check_import_data_for_prefix("id_prefix", "test.tsv", "invalid_base64") + assert len(result['error']) > 0 + + with patch('weko_authors.utils.unpackage_and_check_import_file_for_prefix') as mock_file: + unicode_error = UnicodeDecodeError('utf-8', b'\x80abc', 0, 1, 'invalid start byte') + mock_file.side_effect = unicode_error + result = check_import_data_for_prefix("id_prefix", "test.tsv", "dGVzdA==") + assert len(result['error']) > 0 + + with patch('weko_authors.utils.unpackage_and_check_import_file_for_prefix') as mock_file, \ + patch('weko_authors.utils.validate_import_data_for_prefix') as mock_data: + mock_file.return_value = data + mock_data.side_effect = Exception({"error_msg": "Mocked exception occurred"}) + result = check_import_data_for_prefix("id_prefix", "test.tsv", "dGVzdA==") + assert len(result['error']) > 0 + + +# def validate_import_data_for_prefix(file_data, target): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_validate_import_data_for_prefix -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_validate_import_data_for_prefix(authors_prefix_settings): + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': '', 'status': 'update', 'id': 2}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': 'ISNI', 'name': 'ISNI', 'url': 'http://isni.org', 'is_deleted': ''}], target = 'affiliation_id') + assert result == [{'scheme': 'ISNI', 'name': 'ISNI', 'url': 'http://isni.org', 'is_deleted': '', 'status': 'new'}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': '', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': '', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': '', 'status': 'new', 'errors': ['Scheme is required item.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': 'WEKO', 'name': 'WEKO', 'url': 'http://weko.org', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': 'WEKO', 'name': 'WEKO', 'url': 'http://weko.org', 'is_deleted': '', 'status': 'new', 'errors': ['The scheme WEKO cannot be used.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': 'ORCID', 'name': '', 'url': 'http://orcid.org', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': 'ORCID', 'name': '', 'url': 'http://orcid.org', 'is_deleted': '', 'id': 2, 'status': 'update', 'errors': ['Name is required item.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': '', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': '', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': '', 'status': 'new', 'errors': ['Scheme is required item.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': 'WEKO', 'name': 'WEKO', 'url': 'http://weko.org', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': 'WEKO', 'name': 'WEKO', 'url': 'http://weko.org', 'is_deleted': '', 'status': 'new', 'errors': ['The scheme WEKO cannot be used.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': 'ORCID', 'name': '', 'url': 'http://orcid.org', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': 'ORCID', 'name': '', 'url': 'http://orcid.org', 'is_deleted': '', 'id': 2, 'status': 'update', 'errors': ['Name is required item.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'orcid', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'orcid', 'is_deleted': '', 'id': 2, 'status': 'update', 'errors': ['URL is not URL format.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = [] + result = validate_import_data_for_prefix([{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': 'D'}], target = 'id_prefix') + assert result == [{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': 'D', 'errors': ['The specified scheme does not exist.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_used_scheme_of_id_prefix') as mock_used,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + mock_used.return_value = ( + ["ORCID"], + {1: 'WEKO', 2: 'ORCID', 3: 'CiNii', 4: 'KAKEN2', 5: 'ROR'} + ) + result = validate_import_data_for_prefix([{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': 'D'}], target = 'id_prefix') + assert result == [{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': 'D', 'id': 2, 'status': 'deleted', 'errors': ['The specified scheme is used in the author ID.']}] + + with patch('weko_authors.utils.WekoAuthors.get_scheme_of_id_prefix') as mock_prefix,\ + patch('weko_authors.utils.WekoAuthors.get_scheme_of_affiliaiton_id'): + mock_prefix.return_value = ['ORCID'] + result = validate_import_data_for_prefix([{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': ''}, {'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': ''}], target = 'id_prefix') + assert result == [{'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': '', 'id': 2, 'status': 'update'}, {'scheme': 'ORCID', 'name': 'ORCID', 'url': 'http://orcid.org', 'is_deleted': '', 'id': 2, 'status': 'update', 'errors': ['The specified scheme is duplicated.']}] + + +# def import_id_prefix_to_system(id_prefix): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_import_id_prefix_to_system -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_import_id_prefix_to_system(authors_prefix_settings): + with patch('weko_authors.utils.AuthorsPrefixSettings.create') as mock_create: + import_id_prefix_to_system({'scheme': 'test_scheme', 'status': 'new'}) + mock_create.assert_called_once() + + with patch('weko_authors.utils.AuthorsPrefixSettings.update') as mock_update: + import_id_prefix_to_system({'scheme': 'test_scheme', 'status': 'update', 'id': 1}) + mock_update.assert_called_once() + + with patch('weko_authors.utils.AuthorsPrefixSettings.delete') as mock_delete: + import_id_prefix_to_system({'scheme': 'test_scheme', 'status': 'deleted', 'id': 1}) + mock_delete.assert_called_once() + + with patch('weko_authors.utils.WekoAuthors.get_used_scheme_of_id_prefix') as mock_used: + mock_used.return_value = ( + ["WEKO"], + {1: 'WEKO', 2: 'ORCID', 3: 'CiNii', 4: 'KAKEN2', 5: 'ROR'} + ) + with pytest.raises(Exception) as e: + import_id_prefix_to_system({'scheme': 'WEKO', 'status': 'deleted', 'id': 1}) + assert e.value.args[0] == {'error_id': 'delete_author_link'} + + with pytest.raises(Exception) as e: + import_id_prefix_to_system({'scheme': 'test_scheme', 'status': 'invalid_status'}) + assert e.value.args[0] == {'error_id': 'status_error'} + + with patch('weko_authors.utils.AuthorsPrefixSettings.create') as mock_create: + mock_create.side_effect = [SQLAlchemyError("SQLAlchemyError")] + with pytest.raises(Exception): + import_id_prefix_to_system({'scheme': 'test_scheme', 'status': 'new'}) + mock_create.assert_called_once() + + with patch('weko_authors.utils.AuthorsPrefixSettings.create') as mock_create: + mock_create.side_effect = [TimeoutError("Timeout")] + with pytest.raises(Exception): + import_id_prefix_to_system({'scheme': 'test_scheme', 'status': 'new'}) + mock_create.assert_called_once() + + +# def import_affiliation_id_to_system(affiliation_id): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_import_affiliation_id_to_system -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_import_affiliation_id_to_system(authors_affiliation_settings): + with patch('weko_authors.utils.AuthorsAffiliationSettings.create') as mock_create: + import_affiliation_id_to_system({'scheme': 'test_scheme', 'status': 'new'}) + mock_create.assert_called_once() + + with patch('weko_authors.utils.AuthorsAffiliationSettings.update') as mock_update: + import_affiliation_id_to_system({'scheme': 'test_scheme', 'status': 'update', 'id': 1}) + mock_update.assert_called_once() + + with patch('weko_authors.utils.AuthorsAffiliationSettings.delete') as mock_delete: + import_affiliation_id_to_system({'scheme': 'test_scheme', 'status': 'deleted', 'id': 1}) + mock_delete.assert_called_once() + + with patch('weko_authors.utils.WekoAuthors.get_used_scheme_of_affiliation_id') as mock_used: + mock_used.return_value = ( + ["WEKO"], + {1: 'WEKO', 2: 'ORCID', 3: 'CiNii', 4: 'KAKEN2', 5: 'ROR'} + ) + with pytest.raises(Exception) as e: + import_affiliation_id_to_system({'scheme': 'WEKO', 'status': 'deleted', 'id': 1}) + assert e.value.args[0] == {'error_id': 'delete_author_link'} + + with pytest.raises(Exception) as e: + import_affiliation_id_to_system({'scheme': 'test_scheme', 'status': 'invalid_status'}) + assert e.value.args[0] == {'error_id': 'status_error'} + + with patch('weko_authors.utils.AuthorsAffiliationSettings.create') as mock_create: + mock_create.side_effect = [SQLAlchemyError("SQLAlchemyError")] + with pytest.raises(Exception): + import_affiliation_id_to_system({'scheme': 'test_scheme', 'status': 'new'}) + mock_create.assert_called_once() + + with patch('weko_authors.utils.AuthorsAffiliationSettings.create') as mock_create: + mock_create.side_effect = [TimeoutError("Timeout")] + with pytest.raises(Exception): + import_affiliation_id_to_system({'scheme': 'test_scheme', 'status': 'new'}) + mock_create.assert_called_once() + + +# def band_check_file_for_user(max_page): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_band_check_file_for_user -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_band_check_file_for_user(authors_affiliation_settings): + data = [ + {"pk_id": 1, "authorNameInfo":[{"familyName": "aaa", "firstName": "bbb"}]}, + ] + with patch('weko_authors.utils.get_check_base_name') as mock_check,\ + patch("weko_authors.utils.open") as mock_open,\ + patch("weko_authors.utils.json.load") as mock_json: + # update_cache_data(current_app.config.get("WEKO_AUTHORS_IMPORT_BATCH_SIZE"), 100) + mock_check.return_value = "test_over_max-check" + mock_json.return_value = data + result = band_check_file_for_user(1) + assert result == "var/tmp/authors_import/import_author_check_result_{}.tsv".format(datetime.datetime.now().strftime("%Y%m%d%H%M")) + + data_2 = [ + {"pk_id": 1, "authorNameInfo":[{"familyName": "a", "firstName": ""}, {"familyName": "b", "firstName": ""}],}, + ] + with patch('weko_authors.utils.get_check_base_name') as mock_check,\ + patch("weko_authors.utils.open") as mock_open,\ + patch("weko_authors.utils.json.load") as mock_json: + # update_cache_data(current_app.config.get("WEKO_AUTHORS_IMPORT_BATCH_SIZE"), 100) + mock_check.return_value = "test_over_max-check" + mock_json.return_value = data_2 + result = band_check_file_for_user(1) + assert result == "var/tmp/authors_import/import_author_check_result_{}.tsv".format(datetime.datetime.now().strftime("%Y%m%d%H%M")) + + with patch('weko_authors.utils.get_check_base_name') as mock_check,\ + patch("weko_authors.utils.open") as mock_open: + mock_check.return_value = "test_over_max-check" + mock_open.side_effect = Exception('not found') + with pytest.raises(Exception) as ex: + result = band_check_file_for_user(1) + assert str(ex.value) == 'not found' + + +# def prepare_import_data(max_page_for_import_tab): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_prepare_import_data -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_prepare_import_data(authors_affiliation_settings): + data = [ + {"pk_id": 1, "authorNameInfo":[{"familyName": "aaa", "firstName": "bbb"}]}, + ] + with patch('weko_authors.utils.get_check_base_name') as mock_check,\ + patch("weko_authors.utils.open") as mock_open,\ + patch("weko_authors.utils.json.load") as mock_json: + mock_check.return_value = "test_over_max-check" + mock_json.return_value = data + result = prepare_import_data(1) + assert result[1]== {} + assert result[2]== 1 + + data = [ + {"warnings": 1, "is_deleted": 1, "authorNameInfo":[{"familyName": "aaa", "firstName": "bbb"}]}, + ] + with patch('weko_authors.utils.get_check_base_name') as mock_check,\ + patch("weko_authors.utils.open") as mock_open,\ + patch("weko_authors.utils.json.load") as mock_json: + mock_check.return_value = "test_over_max-check" + mock_json.return_value = data + result = prepare_import_data(3) + assert result[1]== {} + assert result[2]== 3 + + data = [ + {"errors": "error", "pk_id": 1, "authorNameInfo":[{"familyName": "aaa", "firstName": "bbb"}]}, + {"pk_id": 1, "authorNameInfo":[{"familyName": "aaa", "firstName": "bbb"}]}, + ] + with patch('weko_authors.utils.get_check_base_name') as mock_check,\ + patch("weko_authors.utils.open") as mock_open,\ + patch("weko_authors.utils.json.load") as mock_json: + mock_check.return_value = "test_over_max-check" + mock_json.return_value = data + result = prepare_import_data(1) + assert result[1]== {} + assert result[2]== 1 + + data = [ + {"pk_id": 1, "authorNameInfo":[{"familyName": "aaa", "firstName": "bbb"}]}, + ] + with patch('weko_authors.utils.get_check_base_name') as mock_check,\ + patch("weko_authors.utils.open") as mock_open,\ + patch("weko_authors.utils.json.load") as mock_json,\ + patch('weko_authors.utils.current_app.config.get') as mock_max: + mock_check.return_value = "test_over_max-check" + mock_json.return_value = data * 3 + mock_max.side_effect = [ + 2, + "var/tmp/authors_import" + ] + result = prepare_import_data(3) + assert result[0] == data * 2 + assert result[1]['part_number'] == 1 + assert result[1]['count'] == 2 + + # 正常系 + # 条件: リストの中にNoneや空文字が含まれている + # 入力: [None, '', 'valid', {'key': ''}, {'key': 'value'}] + # 期待結果: ['valid', {'key': 'value'}] + def test_clean_deep_list_with_none_and_empty_string(self, app): + data = [None, '', 'valid', {'key': ''}, {'key': 'value'}] + expected = ['valid', {'key': 'value'}] + assert clean_deep(data) == expected + + # 正常系 + # 条件: リストの中に空の辞書が含まれている + # 入力: [{'key': 'value'}, {}] + # 期待結果: [{'key': 'value'}] + def test_clean_deep_list_with_empty_dict(self, app): + data = [{'key': 'value'}, {}] + expected = [{'key': 'value'}] + assert clean_deep(data) == expected + + # 正常系 + # 条件: 辞書の中にNoneや空文字が含まれていない + # 入力: {'fullname': 'Jane Doe', 'email': {"test2": "test2"}, 'test': [{"test2": "test2"}]} + # 期待結果: {'fullname': 'Jane Doe', 'email': {"test2": "test2"}, 'test': [{"test2": "test2"}]} + def test_clean_deep_no_none_or_empty_string(self, app): + data = {'fullname': 'Jane Doe', 'email': {"test2": "test2"}, 'test': [{"test2": "test2"}]} + expected = {'fullname': 'Jane Doe', 'email': {"test2": "test2"}, 'test': [{"test2": "test2"}]} + assert clean_deep(data) == expected + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestUpdateDataForWekoLink -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestUpdateDataForWekoLink: + """update_data_for_weko_linkのテストクラス""" + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::TestUpdateDataForWekoLink::test_update_data_normal_case -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp + def test_update_data_normal_case(self, app): + """ + 正常系 + 条件：weko_linkの内容が更新される場合 + 入力： + - data: nameIdentifiersを含むメタデータ + - weko_link: 更新前のweko_link + 期待結果： + - weko_linkが更新される + - dataのnameIdentifierが更新される + """ + # テスト用データ + data = { + "creators": [ + { + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "old_weko_id_1" + } + ] + } + ] + } + weko_link = {"1": "old_weko_id_1"} + + # AuthorsクラスのMock + author_mock = { + "authorIdInfo": [ + {"idType": "1", "authorId": "new_weko_id_1"} + ] + } + + with patch.object(Authors, 'get_author_by_id', return_value=author_mock): + update_data_for_weko_link(data, weko_link) + result = data["creators"][0]["nameIdentifiers"][0]["nameIdentifier"] + # 検証 + assert result == "new_weko_id_1" + + def test_no_change_in_weko_link(self, app): + """ + 正常系 + 条件：weko_linkの内容が変更されない場合 + 入力： + - data: nameIdentifiersを含むメタデータ + - weko_link: 更新前のweko_link + 期待結果： + - weko_linkは変更されない + - dataは変更されない + """ + data = { + "creators": [ + { + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "weko_id_1" + } + ] + } + ] + } + weko_link = {"1": "weko_id_1"} + data_copy = copy.deepcopy(data) + + author_mock = { + "authorIdInfo": [ + {"idType": "1", "authorId": "weko_id_1"} + ] + } + + with patch.object(Authors, 'get_author_by_id', return_value=author_mock): + update_data_for_weko_link(data, weko_link) + + assert data == data_copy + + def test_author_not_found(self, app): + """ + 正常系 + 条件：Authorsテーブルに該当するauthorが存在しない場合 + 入力： + - data: 任意のメタデータ + - weko_link: 更新前のweko_link + 期待結果： + - weko_linkは変更されない + - dataは変更されない + """ + data = {"creators": [{"name": "test"}]} + weko_link = {"1": "weko_id_1"} + data_copy = copy.deepcopy(data) + weko_link_copy = copy.deepcopy(weko_link) + + with patch.object(Authors, 'get_author_by_id', return_value=None): + update_data_for_weko_link(data, weko_link) + + assert data == data_copy + + def test_id_type_not_1(self, app): + """ + 正常系 + 条件：authorIdInfoのidTypeが1でない場合 + 入力： + - data: 任意のメタデータ + - weko_link: 更新前のweko_link + 期待結果： + - weko_linkは変更されない + - dataは変更されない + """ + data = {"creators": [{"name": "test"}]} + weko_link = {"1": "weko_id_1"} + data_copy = copy.deepcopy(data) + weko_link_copy = copy.deepcopy(weko_link) + + author_mock = { + "authorIdInfo": [ + {"idType": "2", "authorId": "other_id"} + ] + } + + with patch.object(Authors, 'get_author_by_id', return_value=author_mock): + update_data_for_weko_link(data, weko_link) + + assert data == data_copy + + def test_multiple_authors_and_identifiers(self, app): + """ + 正常系 + 条件：複数のauthorとnameIdentifiersがある場合 + 入力： + - data: 複数のauthorとnameIdentifiersを含むメタデータ + - weko_link: 複数のエントリを持つweko_link + 期待結果： + - dataの全てのnameIdentifierが更新される + """ + data = { + "creators": [ + { + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "old_weko_id_1" + }, + { + "nameIdentifierScheme": "OTHER", + "nameIdentifier": "other_id" + } + ] + }, + { + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "old_weko_id_2" + } + ] + } + ], + "contributors": [ + { + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "old_weko_id_3" + } + ] + } + ] + } + weko_link = { + "1": "old_weko_id_1", + "2": "old_weko_id_2", + "3": "old_weko_id_3" + } + + def mock_get_author_by_id(pk_id): + if pk_id == "1": + return {"authorIdInfo": [{"idType": "1", "authorId": "new_weko_id_1"}]} + elif pk_id == "2": + return {"authorIdInfo": [{"idType": "1", "authorId": "new_weko_id_2"}]} + elif pk_id == "3": + return {"authorIdInfo": [{"idType": "1", "authorId": "new_weko_id_3"}]} + return None + + with patch.object(Authors, 'get_author_by_id', side_effect=mock_get_author_by_id): + update_data_for_weko_link(data, weko_link) + + assert data["creators"][0]["nameIdentifiers"][0]["nameIdentifier"] == "new_weko_id_1" + assert data["creators"][1]["nameIdentifiers"][0]["nameIdentifier"] == "new_weko_id_2" + assert data["contributors"][0]["nameIdentifiers"][0]["nameIdentifier"] == "new_weko_id_3" + # 他のスキームのIDは変更されないことを確認 + assert data["creators"][0]["nameIdentifiers"][1]["nameIdentifier"] == "other_id" + + def test_non_list_data_fields(self, app): + """ + 正常系 + 条件：dataのフィールドがリストでない場合 + 入力： + - data: リスト以外のデータ型を含むフィールドを持つメタデータ + - weko_link: weko_link + 期待結果： + - dataの該当フィールドはスキップされる + """ + data = { + "creators": [ + { + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "old_weko_id_1" + } + ] + } + ], + "title": "Test Title", # 文字列フィールド + "description": {"text": "Test Description"} # 辞書フィールド + } + weko_link = {"1": "old_weko_id_1"} + + author_mock = { + "authorIdInfo": [ + {"idType": "1", "authorId": "new_weko_id_1"} + ] + } + + with patch.object(Authors, 'get_author_by_id', return_value=author_mock): + update_data_for_weko_link(data, weko_link) + + assert data["creators"][0]["nameIdentifiers"][0]["nameIdentifier"] == "new_weko_id_1" + # 文字列や辞書フィールドは変更されないこと + assert data["title"] == "Test Title" + assert data["description"] == {"text": "Test Description"} + + def test_string_items_in_list(self, app): + """ + 正常系 + 条件：dataのリストフィールド内に文字列アイテムがある場合 + 入力： + - data: リスト内に文字列アイテムを含むメタデータ + - weko_link:　weko_link + 期待結果： + - 文字列アイテムはスキップされる + """ + data = { + "creators": [ + { + "other_identifiers":[], + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "old_weko_id_1" + } + ] + }, + "Simple String Creator" # 文字列アイテム + ] + } + weko_link = {"1": "old_weko_id_1"} + + author_mock = { + "authorIdInfo": [ + {"idType": "1", "authorId": "new_weko_id_1"} + ] + } + + with patch.object(Authors, 'get_author_by_id', return_value=author_mock): + update_data_for_weko_link(data, weko_link) + + assert data["creators"][0]["nameIdentifiers"][0]["nameIdentifier"] == "new_weko_id_1" + # 文字列アイテムは変更されないこと + assert data["creators"][1] == "Simple String Creator" + + def test_no_matching_nameIdentifier(self, app): + """ + 正常系 + 条件：nameIdentifierがweko_linkの値と一致しない場合 + 入力： + - data: weko_linkと一致しないnameIdentifierを含むメタデータ + - weko_link: weko_link + 期待結果： + - dataのnameIdentifierは更新されない + """ + data = { + "creators": [ + { + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "different_weko_id" + } + ] + } + ] + } + weko_link = {"1": "old_weko_id_1"} + data_copy = copy.deepcopy(data) + + author_mock = { + "authorIdInfo": [ + {"idType": "1", "authorId": "new_weko_id_1"} + ] + } + + with patch.object(Authors, 'get_author_by_id', return_value=author_mock): + update_data_for_weko_link(data, weko_link) + + # 一致するnameIdentifierがないため、データは変更されない + assert data == data_copy + + def test_empty_input_data(self, app): + """ + 正常系 + 条件：空のデータ辞書が入力された場合 + 入力： + - data: 空の辞書 + - weko_link: weko_link + 期待結果： + - dataは変更されない（空のまま） + """ + data = {} + weko_link = {"1": "old_weko_id_1"} + + author_mock = { + "authorIdInfo": [ + {"idType": "1", "authorId": "new_weko_id_1"} + ] + } + + with patch.object(Authors, 'get_author_by_id', return_value=author_mock): + update_data_for_weko_link(data, weko_link) + + assert data == {} + + def test_empty_weko_link(self, app): + """ + 正常系 + 条件：空のweko_linkが入力された場合 + 入力： + - data: 任意のメタデータ + - weko_link: 空の辞書 + 期待結果： + - dataは変更されない + """ + data = { + "creators": [ + { + "nameIdentifiers": [ + { + "nameIdentifierScheme": "WEKO", + "nameIdentifier": "weko_id_1" + } + ] + } + ] + } + weko_link = {} + data_copy = copy.deepcopy(data) + + update_data_for_weko_link(data, weko_link) + + assert data == data_copy + +# def write_to_tempfile(start, row_header, row_label_en, row_label_jp, row_data): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_write_to_tempfile -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_write_to_tempfile(app, mocker): + start = 0 + row_header = ["header1", "header2"] + row_label_en = ["label_en1", "label_en2"] + row_label_jp = ["label_jp1", "label_jp2"] + row_data = [["data1", "data2"], ["data3", "data4"]] + current_cache.delete("weko_authors_export_temp_file_path_key") + mock_open = mocker.patch("builtins.open", mocker.mock_open()) + write_to_tempfile(start, row_header, row_label_en, row_label_jp, row_data) + mock_open.assert_called_with(None, 'a', newline='', encoding='utf-8-sig') + current_cache.set("weko_authors_export_temp_file_path_key",{"key":"authors_export_temp_file_path_key"}) + start = 1 + write_to_tempfile(start, row_header, row_label_en, row_label_jp, row_data) + mock_open.assert_called_with({"key":"authors_export_temp_file_path_key"}, 'a', newline='', encoding='utf-8-sig') + mock_logger = mocker.patch("weko_authors.utils.current_app.logger") + mock_open.side_effect = Exception("File write error") + write_to_tempfile(start, row_header, row_label_en, row_label_jp, row_data) + mock_logger.error.assert_called() + +# def create_result_file_for_user(json): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_utils.py::test_create_result_file_for_user -vv -s --cov-branch --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_create_result_file_for_user(app, mocker): + json = [ + { + "No.": "1", + "Start Date": "2025-01-01", + "End Date": "2025-01-02", + "Previous WEKO ID": "123", + "New WEKO ID": "456", + "full_name": "テスト 太郎", + "Status": "success", + } + ] + mock_result_over_max_data = [ + { + "No.": "1", + "Start Date": "2025-01-01", + "End Date": "2025-01-02", + "Previous WEKO ID": "123", + "New WEKO ID": "456", + "full_name": "テスト 太郎", + "Status": "success", + } + ] + result_path_key = current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_OVER_MAX_FILE_PATH_KEY"] + current_cache.set(result_path_key,{"key":"cache_result_over_max_file_path_key"}) + mocker.patch("builtins.open", mock_open(read_data="")) + mocker.patch("csv.writer", return_value=MagicMock()) + mocker.patch("csv.reader", return_value=iter(mock_result_over_max_data)) + create_result_file_for_user(json) + open.assert_any_call({"key":"cache_result_over_max_file_path_key"}, "r", encoding="utf-8") + csv_writer = csv.writer.return_value + csv_writer.writerow.assert_any_call(["No.", "Start Date", "End Date", "Previous WEKO ID", "New WEKO ID", "full_name", "Status"]) + csv_writer.writerow.assert_any_call(['1', '2025-01-01', '2025-01-02', '123', '456', 'テスト 太郎', 'success']) + + # Exception + mock_logger = mocker.patch("weko_authors.utils.current_app.logger") + mocker.patch("csv.writer", side_effect=Exception) + create_result_file_for_user(json) + mock_logger.error.assert_called() + + # not result_over_max_file_path is true + current_cache.delete(result_path_key) + res = create_result_file_for_user(json) + assert res == None + diff --git a/modules/weko-authors/tests/test_validation.py b/modules/weko-authors/tests/test_validation.py index 7a9a9aa702..76b01669c3 100644 --- a/modules/weko-authors/tests/test_validation.py +++ b/modules/weko-authors/tests/test_validation.py @@ -1,12 +1,17 @@ from mock import patch - +import pytest from weko_authors.contrib.validation import ( validate_by_extend_validator, validate_required, validate_map, validate_identifier_scheme, - validate_external_author_identifier + validate_external_author_identifier, + validate_affiliation_period_end, + validate_digits_for_wekoid, + validate_affiliation_identifier_scheme, + validate_affiliation_period_start, + check_weko_id_is_exits_for_import ) # .tox/c1/bin/pytest --cov=weko_authors tests/test_validation.py -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -15,17 +20,18 @@ # def validate_by_extend_validator(values=[], validator={}): # .tox/c1/bin/pytest --cov=weko_authors tests/test_validation.py::test_validate_by_extend_validator -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_validate_by_extend_validator(): + item = 'item' values = [{'key': 'authorIdInfo[0].idType', 'reduce_keys': ['authorIdInfo', 0, 'idType'], 'value': 'ORCID'}] validator = { 'class_name': 'weko_authors.contrib.validation', 'func_name': 'validate_identifier_scheme' } with patch("weko_authors.contrib.validation.validate_identifier_scheme", return_value=["test_error"]): - result = validate_by_extend_validator(values, validator) + result = validate_by_extend_validator(item, values, validator) assert result == ["test_error"] # values and validator is none - result = validate_by_extend_validator() + result = validate_by_extend_validator(item) assert result == [] # def validate_required(item, values=[], required={}): @@ -81,17 +87,20 @@ def test_validate_map(): # .tox/c1/bin/pytest --cov=weko_authors tests/test_validation.py::test_validate_identifier_scheme -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_validate_identifier_scheme(authors_prefix_settings): # values is none - result = validate_identifier_scheme() + item = 'item' + result = validate_identifier_scheme(item) assert result == [] # value is in schemes + item = 'item' values = [{"key":"authorIdInfo[0].idType","reduce_keys":["authorIdInfo",0,"idType"],"value":"WEKO"}] - result = validate_identifier_scheme(values) + result = validate_identifier_scheme(item, values) assert result == [] # value is not in schemes + item = 'item' values = [{"key":"authorIdInfo[0].idType","reduce_keys":["authorIdInfo",0,"idType"],"value":"NON-EXIST-SCHEME"}] - result = validate_identifier_scheme(values) + result = validate_identifier_scheme(item, values) assert result == ["Specified Identifier Scheme 'NON-EXIST-SCHEME' does not exist."] # def validate_external_author_identifier(item, values=[], @@ -134,3 +143,249 @@ def test_validate_external_author_identifier(authors): existed_external_authors_id = {"2":{"1234":["2"]}} result = validate_external_author_identifier(item, values, existed_external_authors_id) assert result == "External author identifier exists in DB.
1234" + +# .tox/c1/bin/pytest --cov=weko_authors tests/test_validation.py::TestValidateAffiliationPeriodEnd -vv -s --cov-branch --cov-report=html --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +class TestValidateAffiliationPeriodEnd: + """validate_affiliation_period_end関数のテストクラス""" + + @pytest.fixture + def error_messages(self): + """エラーメッセージを返すフィクスチャ""" + return { + 'format_error': "External Affiliation Period must be in the format: yyyy-MM-dd, blank. {}", + 'period_error': "Period end must be after Period start." + } + + def test_empty_values(self): + """ + 正常系 + 条件：値のリストが空 + 入力：item={}, values=[] + 期待結果：空のエラーリスト + """ + item = {} + values = [] + result = validate_affiliation_period_end(item, values) + assert result == [] + + def test_period_end_none(self): + """ + 正常系 + 条件：periodEndが空の場合 + 入力：item={}, values=[{'value': None, 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：空のエラーリスト + """ + item = {} + values = [{'value': None, 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert result == [] + + def test_period_end_empty_string(self): + """ + 正常系 + 条件：periodEndが空文字の場合 + 入力：item={}, values=[{'value': '', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：空のエラーリスト + """ + item = {} + values = [{'value': '', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert result == [] + + def test_period_end_invalid_format(self): + """ + 異常系 + 条件：periodEndのフォーマットが不正な場合 + 入力：item={}, values=[{'value': '2023/01/01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：フォーマットエラーメッセージを含むリスト + """ + item = {} + values = [{'value': '2023/01/01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert len(result) == 1 + assert "External Affiliation Period must be in the format: yyyy-MM-dd, blank. 2023/01/01" in result + + def test_period_end_valid_format_period_start_none(self): + """ + 正常系 + 条件：periodEndのフォーマットが正しく、periodStartがNoneの場合 + 入力：item={'affiliations': [{'periodStart': None}]}, values=[{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：空のエラーリスト + """ + item = {'affiliations': [{'periodStart': None}]} + values = [{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert result == [] + + def test_period_end_valid_format_period_start_empty(self): + """ + 正常系 + 条件：periodEndのフォーマットが正しく、periodStartが空文字の場合 + 入力：item={'affiliations': [{'periodStart': ''}]}, values=[{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：空のエラーリスト + """ + item = {'affiliations': [{'periodStart': ''}]} + values = [{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert result == [] + + def test_period_start_invalid_format(self): + """ + 異常系 + 条件：periodEndのフォーマットが正しく、periodStartのフォーマットが不正な場合 + 入力：item={'affiliations': [{'periodStart': '2023/01/01'}]}, values=[{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：フォーマットエラーメッセージを含むリスト + """ + item = {'affiliations': [{'periodStart': '2023/01/01'}]} + values = [{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert len(result) == 1 + assert "External Affiliation Period must be in the format: yyyy-MM-dd, blank. 2023-01-01" in result + + def test_period_end_before_period_start(self): + """ + 異常系 + 条件：periodEndがperiodStartより前の日付の場合 + 入力：item={'affiliations': [{'periodStart': '2023-02-01'}]}, values=[{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：期間エラーメッセージを含むリスト + """ + item = {'affiliations': [{'periodStart': '2023-02-01'}]} + values = [{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert len(result) == 1 + assert "Period end must be after Period start." in result + + def test_period_end_equal_period_start(self): + """ + 正常系 + 条件：periodEndとperiodStartが同じ日付の場合 + 入力：item={'affiliations': [{'periodStart': '2023-01-01'}]}, values=[{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：空のエラーリスト（同日は許容される） + """ + item = {'affiliations': [{'periodStart': '2023-01-01'}]} + values = [{'value': '2023-01-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert result == [] + + def test_period_end_after_period_start(self): + """ + 正常系 + 条件：periodEndがperiodStartより後の日付の場合 + 入力：item={'affiliations': [{'periodStart': '2023-01-01'}]}, values=[{'value': '2023-02-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + 期待結果：空のエラーリスト + """ + item = {'affiliations': [{'periodStart': '2023-01-01'}]} + values = [{'value': '2023-02-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}] + result = validate_affiliation_period_end(item, values) + assert result == [] + + def test_multiple_values(self): + """ + 正常系と異常系の混合 + 条件：複数の値がある場合 + 入力：複数の値を持つitem, values + 期待結果：エラーがある項目のみエラーメッセージを含むリスト + """ + item = { + 'affiliations': [ + {'periodStart': '2023-01-01'}, + {'periodStart': '2023-03-01'}, + {'periodStart': '2023/05/01'}, + {'periodStart': None} + ] + } + values = [ + {'value': '2023-02-01', 'reduce_keys': ['affiliations', 0, 'periodEnd']}, + {'value': '2023-02-01', 'reduce_keys': ['affiliations', 1, 'periodEnd']}, + {'value': '2023-06-01', 'reduce_keys': ['affiliations', 2, 'periodEnd']}, + {'value': '2023-07-01', 'reduce_keys': ['affiliations', 3, 'periodEnd']} + ] + result = validate_affiliation_period_end(item, values) + assert len(result) == 2 + assert "Period end must be after Period start." in result + + +# def validate_digits_for_wekoid(items, values=[]) +# .tox/c1/bin/pytest --cov=weko_authors tests/test_validation.py::test_validate_digits_for_wekoid -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_validate_digits_for_wekoid(): + items = 'items' + values = [ + {'value': '111'}, + ] + result = validate_digits_for_wekoid(items, values) + assert result == [] + + values_2 = [ + {'value': ''}, + ] + result = validate_digits_for_wekoid(items, values_2) + assert result == ["WEKO ID is required item."] + + values_3 = [ + {'value': '１１１'}, + ] + result = validate_digits_for_wekoid(items, values_3) + assert result == ["WEKO ID is Half-width digits only"] + + + # def validate_affiliation_identifier_scheme(item, values=[]) +# .tox/c1/bin/pytest --cov=weko_authors tests/test_validation.py::test_validate_affiliation_identifier_scheme -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_validate_affiliation_identifier_scheme(authors_affiliation_settings): + + + item = 'item' + values = [{"key":"authorIdInfo[0].idType","reduce_keys":["authorIdInfo",0,"idType"],"value":"ISNI"}] + result = validate_affiliation_identifier_scheme(item, values) + assert result == [] + + item = 'item' + values_2 = [ + {'value': '123'}, + ] + result = validate_affiliation_identifier_scheme(item, values_2) + assert result == ["Specified Affiliation Identifier Scheme '123' does not exist."] + + +# def validate_affiliation_period_start(item, values=[]) +# .tox/c1/bin/pytest --cov=weko_authors tests/test_validation.py::test_validate_affiliation_period_start -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_validate_affiliation_period_start(authors): + item = 'item' + values = [ + {'value': '2025-3-17'}, + ] + result = validate_affiliation_period_start(item, values) + assert result == [] + + values_2 = [ + {'value': '2025317'}, + ] + result = validate_affiliation_period_start(item, values_2) + assert result == ["External Affiliation Period must be in the format: yyyy-MM-dd, blank. 2025317"] + + +# def check_weko_id_is_exits_for_import(pk_id, weko_id, existed_external_authors_id={}) +# .tox/c1/bin/pytest --cov=weko_authors tests/test_validation.py::test_check_weko_id_is_exits_for_import -vv -s --cov-branch --cov-report=term --cov-report=html --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_check_weko_id_is_exits_for_import(authors): + pk_id = '2' + weko_id = '1' + existed_external_authors_id = { + '1': {}, + } + result = check_weko_id_is_exits_for_import(pk_id, weko_id, existed_external_authors_id) + assert result == [] + + pk_id = '2' + weko_id = '1' + existed_external_authors_id = { + '1': {'1': ["2"],}, + } + result = check_weko_id_is_exits_for_import(pk_id, weko_id, existed_external_authors_id) + assert result == [] + + pk_id = '3' + weko_id = '1' + existed_external_authors_id = { + '1': {'1': ["2"],}, + } + result = check_weko_id_is_exits_for_import(pk_id, weko_id, existed_external_authors_id) + assert result == ["Specified WEKO ID already exist."] \ No newline at end of file diff --git a/modules/weko-authors/tests/test_views.py b/modules/weko-authors/tests/test_views.py index 3514ee757f..2aa6382a19 100644 --- a/modules/weko-authors/tests/test_views.py +++ b/modules/weko-authors/tests/test_views.py @@ -103,82 +103,120 @@ def test_create_acl_users(client, users, index, is_permission): res = client.post(url,content_type='text/plain') assert_role(res, is_permission) -# .tox/c1/bin/pytest --cov=weko_authors tests/test_views.py::test_create -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp -@pytest.mark.parametrize('base_app',[dict( - is_es=True -)],indirect=['base_app']) -def test_create(client, users, esindex): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_views.py::test_create_author -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_create_author(client, users, mocker): """ Test of create author. :param client: The flask client. """ - url = url_for("weko_authors.create") login_user_via_session(client=client, email=users[0]['email']) - input = { - "id": "", - "pk_id": "", - "authorNameInfo": [ - { - "familyName": "テスト", - "firstName": "タロウ", - "fullName": "", - "language": "ja-Kana", - "nameFormat": "familyNmAndNm", - "nameShowFlg": "true" - } - ], - "authorIdInfo": [ - { - "idType": "2", - "authorId": "0123", - "authorIdShowFlg": "true" - } - ], - "emailInfo": [ - {"email": "example@com"} - ] + url = url_for("weko_authors.create") + + # 条件: Content-Typeがapplication/jsonではない場合 + # 入力: Content-Typeがtext/plain + # 期待結果: ステータスコード200、メッセージ"Header Error" + res = client.post(url, content_type='text/plain') + assert res.status_code == 200 + assert get_json(res) == {"msg": "Header Error"} + + # 条件: WEKO IDが設定されていない場合 + # 入力: authorIdInfoにidTypeが1のauthorIdが存在しない + # 期待結果: ステータスコード500、メッセージ"Please set WEKO ID." + input_data = { + "authorIdInfo": [{"idType": "2", "authorId": "0123"}] } - # content_type is not json - res = client.post(url, content_type="plain/text") - assert res.status_code==200 - assert get_json(res) == {"msg":"Header Error"} + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Please set WEKO ID."} - # success create - - es_id = uuid.uuid4() - pk_id=10 - with patch('weko_authors.views.Authors.get_sequence', return_value=pk_id): - with patch("weko_authors.views.uuid.uuid4",return_value=es_id): - res = client.post(url, - data=json.dumps(input), - content_type='application/json') - assert res.status_code==200 - assert get_json(res) == {"msg":"Success"} - author = Authors.query.filter_by(id=pk_id).one() - assert author - assert type(author.json) == dict - res = current_search_client.get(index=current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"],doc_type=current_app.config['WEKO_AUTHORS_ES_DOC_TYPE'],id=str(es_id)) - assert res["_source"]["pk_id"] == str(pk_id) + # 条件: WEKO IDが半角数字でない場合 + # 入力: authorIdInfoにidTypeが1のauthorIdが半角数字でない + # 期待結果: ステータスコード500、メッセージ"Please set the WEKOID in the half digit." + input_data = { + "authorIdInfo": [{"idType": "1", "authorId": "abc"}] + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(False, "not half digit")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Please set the WEKOID in the half digit."} - # raise Exception - es_id = uuid.uuid4() - pk_id=11 - with patch("weko_authors.views.Authors",side_effect=Exception("test_error")): - with patch('weko_authors.views.Authors.get_sequence', return_value=pk_id): - with patch("weko_authors.views.uuid.uuid4",return_value=es_id): - res = client.post(url, - data=json.dumps(input), - content_type='application/json') - - assert res.status_code==500 - assert get_json(res) == {"msg":"Failed"} - assert Authors.query.filter_by(id=pk_id).one_or_none() == None - with pytest.raises(NotFoundError): - res = current_search_client.get(index=current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"],doc_type=current_app.config['WEKO_AUTHORS_ES_DOC_TYPE'],id=str(es_id)) - - + # 条件: WEKO IDが既に存在する場合 + # 入力: authorIdInfoにidTypeが1のauthorIdが既に存在する + # 期待結果: ステータスコード500、メッセージ"The value is already in use as WEKO ID." + input_data = { + "authorIdInfo": [{"idType": "1", "authorId": "123"}] + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(False, "already exists")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "The value is already in use as WEKO ID."} + # 条件: affiliation start dateとend dateがyyyy-MM-dd形式でない場合 + # 入力: affiliation start dateとend dateがyyyy-MM-dd形式でない + # 期待結果: ステータスコード500、メッセージ"Please set the affiliation start date and end date in the format yyyy-MM-dd." + input_data = { + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2021/01/01", "endDate": "2021/12/31"}] + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(True, "")) + mocker.patch('weko_authors.views.check_period_date', return_value=(False, "not date format")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Please set the affiliation start date and end date in the format yyyy-MM-dd."} + + # 条件: affiliation start dateがend dateより後の場合 + # 入力: affiliation start dateがend dateより後 + # 期待結果: ステータスコード500、メッセージ"The end date must be after the start date." + input_data = { + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2022-01-01", "endDate": "2021-12-31"}] + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(True, "")) + mocker.patch('weko_authors.views.check_period_date', return_value=(False, "start is after end")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "The end date must be after the start date."} + + # 条件: 正常な入力 + # 入力: 正常なauthorIdInfoとaffiliationInfo + # 期待結果: ステータスコード200、メッセージ"Success" + input_data = { + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2021-01-01", "endDate": "2021-12-31"}] + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(True, "")) + mocker.patch('weko_authors.views.check_period_date', return_value=(True, "")) + mocker.patch('weko_authors.views.WekoAuthors.create', return_value=None) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 200 + assert get_json(res) == {"msg": "Success"} + # 条件: WekoAuthors.createで例外が発生する場合 + # 入力: 正常なauthorIdInfoとaffiliationInfo + # 期待結果: ステータスコード500、メッセージ"Failed" + input_data = { + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2021-01-01", "endDate": "2021-12-31"}] + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(True, "")) + mocker.patch('weko_authors.views.check_period_date', return_value=(True, "")) + mocker.patch('weko_authors.views.WekoAuthors.create', side_effect=Exception("test_error")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Failed"} + + # 条件: validate_weko_idで例外が発生する場合 + # 入力: 正常なauthorIdInfoとaffiliationInfo + # 期待結果: ステータスコード500、メッセージ"Failed" + input_data = { + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2021-01-01", "endDate": "2021-12-31"}] + } + mocker.patch('weko_authors.views.validate_weko_id', side_effect=Exception("test_error")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Failed"} + # .tox/c1/bin/pytest --cov=weko_authors tests/test_views.py::test_update_author_acl_guest -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp @@ -211,72 +249,150 @@ def test_update_author_acl_users(client, users, index, is_permission): assert_role(res, is_permission) # .tox/c1/bin/pytest --cov=weko_authors tests/test_views.py::test_update_author -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp -@pytest.mark.parametrize('base_app',[dict( - is_es=True -)],indirect=['base_app']) -def test_update_author(client, db, users, esindex, create_author): +def test_update_author(client, users, mocker): """ - Test of update author data. + Test of create author. :param client: The flask client. """ - test_data = { - "authorNameInfo": [{"familyName": "テスト","firstName": "ハナコ","fullName": "","language": "ja-Kana","nameFormat": "familyNmAndNm","nameShowFlg": "true"}], - "authorIdInfo": [{"idType": "2","authorId": "01234","authorIdShowFlg": "true"}], - "emailInfo": [{"email": "example@com"}] - } login_user_via_session(client=client, email=users[0]['email']) url = url_for("weko_authors.update_author") - - # success update - id = 1 - es_id = create_author(json.loads(json.dumps(test_data)), id) - input = { - "id": es_id, - "pk_id": id, - "authorNameInfo": [{"familyName": "テスト","firstName": "タロウ","fullName": "","language": "ja-Kana","nameFormat": "familyNmAndNm","nameShowFlg": "true"}], - "authorIdInfo": [{"idType": "2","authorId": "0123","authorIdShowFlg": "true"}], - "emailInfo": [{"email": "examplechanged@com"}] + + # 条件: Content-Typeがapplication/jsonではない場合 + # 入力: Content-Typeがtext/plain + # 期待結果: ステータスコード200、メッセージ"Header Error" + res = client.post(url, content_type='text/plain') + assert res.status_code == 200 + assert get_json(res) == {"msg": "Header Error"} + + # 条件: WEKO IDが設定されていない場合 + # 入力: authorIdInfoにidTypeが1のauthorIdが存在しない + # 期待結果: ステータスコード500、メッセージ"Please set WEKO ID." + input_data = { + "forceChangeFlag": "false", + "author":{ + "pk_id": "1", + "authorIdInfo": [{"idType": "2", "authorId": "0123"}] + } + } + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Please set WEKO ID."} + + # 条件: WEKO IDが半角数字でない場合 + # 入力: authorIdInfoにidTypeが1のauthorIdが半角数字でない + # 期待結果: ステータスコード500、メッセージ"Please set the WEKOID in the half digit." + input_data = { + "forceChangeFlag": "false", + "author":{ + "pk_id": "1", + "authorIdInfo": [{"idType": "1", "authorId": "abc"}] + } } - with patch('weko_deposit.tasks.update_items_by_authorInfo'): - res = client.post(url, - data=json.dumps(input), - content_type='application/json') - assert res.status_code == 200 - assert get_json(res) == {"msg":"Success"} - author = Authors.query.filter_by(id=id).one() - assert author - assert author.json["authorNameInfo"][0]["firstName"] == "タロウ" - res = current_search_client.get(index=current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"],doc_type=current_app.config['WEKO_AUTHORS_ES_DOC_TYPE'],id=es_id) - assert res["_source"]["authorNameInfo"][0]["firstName"] == "タロウ" + mocker.patch('weko_authors.views.validate_weko_id', return_value=(False, "not half digit")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Please set the WEKOID in the half digit."} - # failed update - id = 2 - es_id = create_author(json.loads(json.dumps(test_data)), id) - input = { - "id": es_id, - "pk_id": id, - "authorNameInfo": [{"familyName": "テスト","firstName": "タロウ","fullName": "","language": "ja-Kana","nameFormat": "familyNmAndNm","nameShowFlg": "true"}], - "authorIdInfo": [{"idType": "2","authorId": "0123","authorIdShowFlg": "true"}], - "emailInfo": [{"email": "examplechanged@com"}] + # 条件: WEKO IDが既に存在する場合 + # 入力: authorIdInfoにidTypeが1のauthorIdが既に存在する + # 期待結果: ステータスコード500、メッセージ"The value is already in use as WEKO ID." + input_data = { + "forceChangeFlag": "false", + "author":{ + "pk_id": "1", + "authorIdInfo": [{"idType": "1", "authorId": "123"}] + } } - - with patch("weko_authors.views.db.session.commit",side_effect=Exception("test_error")): - res = client.post(url, - data=json.dumps(input), - content_type='application/json') - assert res.status_code == 500 - assert get_json(res) == {"msg":"Failed"} - author = Authors.query.filter_by(id=id).one() - assert author - assert author.json["authorNameInfo"][0]["firstName"] == "ハナコ" - res = current_search_client.get(index=current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"],doc_type=current_app.config['WEKO_AUTHORS_ES_DOC_TYPE'],id=es_id) - assert res["_source"]["authorNameInfo"][0]["firstName"] == "ハナコ" - + mocker.patch('weko_authors.views.validate_weko_id', return_value=(False, "already exists")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "The value is already in use as WEKO ID."} - # content_type is not json - res = client.post(url, content_type="plain/text") + # 条件: affiliation start dateとend dateがyyyy-MM-dd形式でない場合 + # 入力: affiliation start dateとend dateがyyyy-MM-dd形式でない + # 期待結果: ステータスコード500、メッセージ"Please set the affiliation start date and end date in the format yyyy-MM-dd." + input_data = { + "forceChangeFlag": "false", + "author":{ + "pk_id": "1", + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2021/01/01", "endDate": "2021/12/31"}] + } + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(True, "")) + mocker.patch('weko_authors.views.check_period_date', return_value=(False, "not date format")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Please set the affiliation start date and end date in the format yyyy-MM-dd."} + + # 条件: affiliation start dateがend dateより後の場合 + # 入力: affiliation start dateがend dateより後 + # 期待結果: ステータスコード500、メッセージ"The end date must be after the start date." + input_data = { + "forceChangeFlag": "false", + "author":{ + "pk_id": "1", + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2022-01-01", "endDate": "2021-12-31"}] + } + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(True, "")) + mocker.patch('weko_authors.views.check_period_date', return_value=(False, "start is after end")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "The end date must be after the start date."} + + # 条件: 正常な入力 + # 入力: 正常なauthorIdInfoとaffiliationInfo + # 期待結果: ステータスコード200、メッセージ"Success" + input_data = { + "forceChangeFlag": "false", + "author":{ + "pk_id": "1", + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2021-01-01", "endDate": "2021-12-31"}] + } + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(True, "")) + mocker.patch('weko_authors.views.check_period_date', return_value=(True, "")) + mocker.patch('weko_authors.views.WekoAuthors.update', return_value=None) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') assert res.status_code == 200 - assert get_json(res) == {"msg":"Header Error"} + assert get_json(res) == {"msg": "Success"} + + # 条件: WekoAuthors.updateで例外が発生する場合 + # 入力: 正常なauthorIdInfoとaffiliationInfo + # 期待結果: ステータスコード500、メッセージ"Failed" + input_data = { + "forceChangeFlag": "false", + "author":{ + "pk_id": "1", + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2021-01-01", "endDate": "2021-12-31"}] + } + } + mocker.patch('weko_authors.views.validate_weko_id', return_value=(True, "")) + mocker.patch('weko_authors.views.check_period_date', return_value=(True, "")) + mocker.patch('weko_authors.views.WekoAuthors.update', side_effect=Exception("test_error")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Failed"} + + # 条件: validate_weko_idで例外が発生する場合 + # 入力: 正常なauthorIdInfoとaffiliationInfo + # 期待結果: ステータスコード500、メッセージ"Failed" + input_data = { + "forceChangeFlag": "false", + "author":{ + "pk_id": "1", + "authorIdInfo": [{"idType": "1", "authorId": "123"}], + "affiliationInfo": [{"startDate": "2021-01-01", "endDate": "2021-12-31"}] + } + } + mocker.patch('weko_authors.views.validate_weko_id', side_effect=Exception("test_error")) + res = client.post(url, data=json.dumps(input_data), content_type='application/json') + assert res.status_code == 500 + assert get_json(res) == {"msg": "Failed"} # .tox/c1/bin/pytest --cov=weko_authors tests/test_views.py::test_delete_author_acl_guest -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_delete_author_acl_guest(client): @@ -397,6 +513,7 @@ def test_get_acl_users(client, users, index, is_permission): res = client.post(url, json={}) assert_role(res, is_permission) + # .tox/c1/bin/pytest --cov=weko_authors tests/test_views.py::test_get -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_get(client, users): """ @@ -411,13 +528,11 @@ def search(self,index=None,doc_type=None, body=None): url = url_for("weko_authors.get") login_user_via_session(client=client, email=users[0]['email']) - # not exist searchKey, sortKey, sortOrder - input = {"searchKey": "", "pageNumber": 1, "numOfPage": 25, - "sortKey": "", "sortOrder": ""} + input = {"searchKey": "example", "pageNumber": 1, "numOfPage": 10, + "sortKey": "name", "sortOrder": "asc"} data = { "test-authors":{"hits":{"hits":[ - {"_source":{"authorIdInfo":[]}}, # author_id_info is false - {"_source":{"authorIdInfo":[{"authorId":"test_id"}]}} + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, # author_id_info is false ]}}, "test-weko":{"hits":{"total":1}} } @@ -425,37 +540,100 @@ def search(self,index=None,doc_type=None, body=None): record_indexer.client=MockClient(data) test = { "hits":{"hits":[ - {"_source":{"authorIdInfo":[]}}, - {"_source":{"authorIdInfo":[{"authorId":"test_id"}]}} + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, ]}, - "item_cnt":{"aggregations":{"item_count":{"buckets":[{"key":"test_id","doc_count":1}]}}} + "item_cnt":{"aggregations":{"item_count":{"buckets":[{"key":"xxx","doc_count":1}]}}} } with patch("weko_authors.views.RecordIndexer",return_value=record_indexer): res = client.post(url, json=input) assert res.status_code == 200 assert get_json(res) == test - - input = {"searchKey": "test_key", "pageNumber": 1, "numOfPage": 25, - "sortKey": "test_sort", "sortOrder": "test_order"} + + input = {"searchKey": "", "pageNumber": 1, "numOfPage": 10, + "sortKey": "name", "sortOrder": "asc"} + data = { + "test-authors":{"hits":{"hits":[ + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, # author_id_info is false + ]}}, + "test-weko":{"hits":{"total":1}} + } + record_indexer = RecordIndexer() + record_indexer.client=MockClient(data) + test = { + "hits":{"hits":[ + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, + ]}, + "item_cnt":{"aggregations":{"item_count":{"buckets":[{"key":"xxx","doc_count":1}]}}} + } + with patch("weko_authors.views.RecordIndexer",return_value=record_indexer): + res = client.post(url, json=input) + assert res.status_code == 200 + assert get_json(res) == test + + input = {"searchKey": "example", "pageNumber": 1, "numOfPage": None, + "sortKey": "name", "sortOrder": "asc"} + data = { + "test-authors":{"hits":{"hits":[ + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, # author_id_info is false + ]}}, + "test-weko":{"hits":{"total":1}} + } + record_indexer = RecordIndexer() + record_indexer.client=MockClient(data) + test = { + "hits":{"hits":[ + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, + ]}, + "item_cnt":{"aggregations":{"item_count":{"buckets":[{"key":"xxx","doc_count":1}]}}} + } + with patch("weko_authors.views.RecordIndexer",return_value=record_indexer): + res = client.post(url, json=input) + assert res.status_code == 200 + assert get_json(res) == test + + input = {"searchKey": "example", "pageNumber": None, "numOfPage": 10, + "sortKey": "name", "sortOrder": "asc"} + data = { + "test-authors":{"hits":{"hits":[ + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, # author_id_info is false + ]}}, + "test-weko":{"hits":{"total":1}} + } + record_indexer = RecordIndexer() + record_indexer.client=MockClient(data) + test = { + "hits":{"hits":[ + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, + ]}, + "item_cnt":{"aggregations":{"item_count":{"buckets":[{"key":"xxx","doc_count":1}]}}} + } + with patch("weko_authors.views.RecordIndexer",return_value=record_indexer): + res = client.post(url, json=input) + assert res.status_code == 200 + assert get_json(res) == test + + input = {"searchKey": "example", "pageNumber": 1, "numOfPage": 10, + "sortKey": "", "sortOrder": ""} data = { "test-authors":{"hits":{"hits":[ - {"_source":{"authorIdInfo":[{"authorId":"test_id"}]}} + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, # author_id_info is false ]}}, - "test-weko":{"hits":{"total":0}}# result_itemCnt is not + "test-weko":{"hits":{"total":1}} } record_indexer = RecordIndexer() record_indexer.client=MockClient(data) test = { "hits":{"hits":[ - {"_source":{"authorIdInfo":[{"authorId":"test_id"}]}} + {"_source":{"authorIdInfo":[{"authorId":"test_id"}], 'pk_id':'xxx'}}, ]}, - "item_cnt":{"aggregations":{"item_count":{"buckets":[]}}} + "item_cnt":{"aggregations":{"item_count":{"buckets":[{"key":"xxx","doc_count":1}]}}} } with patch("weko_authors.views.RecordIndexer",return_value=record_indexer): res = client.post(url, json=input) assert res.status_code == 200 assert get_json(res) == test + # .tox/c1/bin/pytest --cov=weko_authors tests/test_views.py::test_getById_acl_guest -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp def test_getById_acl_guest(client): """ @@ -1301,4 +1479,90 @@ def test_dbsession_clean(app, db): itemtype_name3 = ItemTypeName(id=3,name="テスト3",has_site_license=True, is_active=True) db.session.add(itemtype_name3) dbsession_clean(Exception) - assert ItemTypeName.query.filter_by(id=3).first() is None \ No newline at end of file + assert ItemTypeName.query.filter_by(id=3).first() is None + + +# def get_max_weko_id(): +# .tox/c1/bin/pytest --cov=weko_authors tests/test_views.py::test_get_max_weko_id -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-authors/.tox/c1/tmp +def test_get_max_weko_id(client, users, mocker ): + class MockClient(): + def __init__(self,data): + self.data = data + def search(self,index=None,body=None,scroll='2m'): + return self.data[index] + def scroll(self,scroll_id='scroll_id',scroll='2m'): + self.data['test-authors']['hits'] ={"hits":{"hits":[]}} + return self.data['test-authors']['hits'] + + url = url_for("weko_authors.get_max_weko_id") + login_user_via_session(client=client, email=users[0]['email']) + + data = { + "test-authors": { + "hits": { + "hits": [ + {"_source": {"authorIdInfo": [{"authorId": "2",'idType': '1'}]}, 'pk_id': 'xxx'} + ] + }, + "_scroll_id": "AAA" + }, + "test-weko": { + "hits": { + "total": 1 + } + } + } + + record_indexer = RecordIndexer() + record_indexer.client=MockClient(data) + mocker.patch("weko_authors.views.RecordIndexer",return_value=record_indexer) + test = {'max_author_id': 2} + res = client.get(url) + assert get_json(res) == test + + data_2 = { + "test-authors": { + "hits": { + "hits": [ + {"_source": {}, 'pk_id': 'xxx'} + ] + }, + "_scroll_id": "AAA" + }, + "test-weko": { + "hits": { + "total": 1 + } + } + } + + record_indexer = RecordIndexer() + record_indexer.client=MockClient(data_2) + mocker.patch("weko_authors.views.RecordIndexer",return_value=record_indexer) + test = {'max_author_id': 0} + res = client.get(url) + assert get_json(res) == test + + + data_3 = { + "test-authors": { + "hits": { + "hits": [ + {"_source": {"authorIdInfo": [{"authorId": "2",'idType': '1'}]}, 'pk_id': 'xxx'} + ] + }, + "_scroll_id": "AAA" + }, + "test-weko": { + "hits": { + "total": 1 + } + } + } + + record_indexer = RecordIndexer() + record_indexer.client=MockClient(data_3) + mocker.patch("weko_authors.views.RecordIndexer",return_value=record_indexer) + test = {'max_author_id': 0} + res = client.get(url) + assert get_json(res) == test \ No newline at end of file diff --git a/modules/weko-authors/tox.result.gz b/modules/weko-authors/tox.result.gz index 2eb6482681..d68f063b7e 100644 Binary files a/modules/weko-authors/tox.result.gz and b/modules/weko-authors/tox.result.gz differ diff --git a/modules/weko-authors/weko_authors/__init__.py b/modules/weko-authors/weko_authors/__init__.py index ce0f90579c..14d51b604c 100644 --- a/modules/weko-authors/weko_authors/__init__.py +++ b/modules/weko-authors/weko_authors/__init__.py @@ -20,7 +20,7 @@ """Module of weko-authors.""" -from .ext import WekoAuthors +from .ext import WekoAuthors, WekoAuthorsREST from .version import __version__ -__all__ = ('__version__', 'WekoAuthors') +__all__ = ('__version__', 'WekoAuthors', 'WekoAuthorsREST') \ No newline at end of file diff --git a/modules/weko-authors/weko_authors/admin.py b/modules/weko-authors/weko_authors/admin.py index d889fd49f3..5c0634528d 100644 --- a/modules/weko-authors/weko_authors/admin.py +++ b/modules/weko-authors/weko_authors/admin.py @@ -22,23 +22,25 @@ from __future__ import absolute_import, print_function -import json +import os +import json, tempfile, datetime, base64 from celery import group, states from celery.task.control import revoke -from flask import abort, current_app, request, session +from flask import abort, current_app, request, session, send_file from flask.helpers import url_for from flask.json import jsonify from flask_admin import BaseView, expose from flask_babelex import gettext as _ from invenio_files_rest.models import FileInstance -from weko_workflow.utils import update_cache_data +from invenio_cache import current_cache from .config import WEKO_AUTHORS_EXPORT_FILE_NAME, \ WEKO_AUTHORS_IMPORT_CACHE_KEY from .permissions import author_permission -from .tasks import check_is_import_available, export_all, import_author -from .utils import check_import_data, delete_export_status, \ - get_export_status, get_export_url, set_export_status +from .tasks import check_is_import_available, export_all, import_author, import_id_prefix, import_affiliation_id, import_author_over_max +from .utils import check_import_data, check_import_data_for_prefix, delete_export_status, \ + get_export_status, get_export_url, set_export_status, check_file_name, delete_export_url,\ + band_check_file_for_user, prepare_import_data, create_result_file_for_user, update_cache_data class AuthorManagementView(BaseView): @@ -107,8 +109,13 @@ def download(self): data = get_export_url() if data.get('file_uri'): file_instance = FileInstance.get_by_uri(data.get('file_uri')) - file_name = WEKO_AUTHORS_EXPORT_FILE_NAME + '.' + \ + export_target = current_cache.get(current_app.config['WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY']) + base_file_name = check_file_name(export_target) + file_name = "{}_{}.{}".format( + base_file_name, + file_instance.updated.strftime("%Y%m%d%H%M"), current_app.config.get('WEKO_ADMIN_OUTPUT_FORMAT', 'tsv').lower() + ) return file_instance.send_file( file_name, mimetype='application/octet-stream', @@ -121,6 +128,10 @@ def download(self): @expose('/check_status', methods=['GET']) def check_status(self): """Api check export status.""" + + # stop_pointを確認 + stop_point= current_cache.get(current_app.config["WEKO_AUTHORS_EXPORT_CACHE_STOP_POINT_KEY"]) + status = get_export_status() if not status: status = get_export_url() @@ -130,14 +141,27 @@ def check_status(self): or task.state == states.REVOKED: delete_export_status() status = get_export_url() - if not task.result: + if not task.result and not stop_point: status['error'] = 'export_fail' else: status['file_uri'] = get_export_url().get('file_uri', '') + if stop_point: + status['stop_point'] = stop_point # set download_link status['download_link'] = url_for( 'authors/export.download', _external=True) + status['filename'] = '' + file_instance = FileInstance.get_by_uri(status.get('file_uri', '')) + if file_instance: + # export_targetによってfilenameを変更 + export_target = current_cache.get(current_app.config['WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY']) + base_file_name = check_file_name(export_target) + status['filename'] = "{}_{}.{}".format( + base_file_name, + file_instance.updated.strftime("%Y%m%d%H%M"), + current_app.config.get('WEKO_ADMIN_OUTPUT_FORMAT', 'tsv').lower() + ) if not status.get('file_uri'): status['download_link'] = '' if 'file_uri' in status: @@ -152,7 +176,26 @@ def check_status(self): @expose('/export', methods=['POST']) def export(self): """Process export authors.""" - task = export_all.delay() + data = request.get_json() + export_target = data.get("isTarget","") + #実行時に以前のexport_urlを削除 + delete_export_url() + if export_target == "author_db": + # 一時ファイルの作成 + temp_folder_path = current_app.config.get("WEKO_AUTHORS_EXPORT_TEMP_FOLDER_PATH") + os.makedirs(temp_folder_path, exist_ok=True) + prefix = current_app.config["WEKO_AUTHORS_EXPORT_TMP_PREFIX"] + datetime.datetime.now().strftime("%Y%m%d%H%M") + + with tempfile.NamedTemporaryFile(dir=temp_folder_path, prefix=prefix, suffix='.tsv', mode='w+', delete=False) as temp_file: + temp_file_path = temp_file.name + + # redisに一時ファイルのパスを保存 + update_cache_data( + current_app.config["WEKO_AUTHORS_EXPORT_CACHE_TEMP_FILE_PATH_KEY"], + temp_file_path, + current_app.config["WEKO_AUTHORS_CACHE_TTL"] + ) + task = export_all.delay(export_target) set_export_status(task_id=task.id) return jsonify({ 'code': 200, @@ -166,6 +209,14 @@ def cancel(self): result = {'status': 'fail'} try: status = get_export_status() + # stop_pointがあるならstop_pointとtemp_file_pathを削除 + if current_cache.get(current_app.config["WEKO_AUTHORS_EXPORT_CACHE_STOP_POINT_KEY"]): + current_cache.delete(current_app.config["WEKO_AUTHORS_EXPORT_CACHE_STOP_POINT_KEY"]) + temp_file_path=current_cache.get(\ + current_app.config["WEKO_AUTHORS_EXPORT_CACHE_TEMP_FILE_PATH_KEY"]) + if temp_file_path: + os.remove(temp_file_path) + current_cache.delete(current_app.config["WEKO_AUTHORS_EXPORT_CACHE_TEMP_FILE_PATH_KEY"]) if status and status.get('task_id'): revoke(status.get('task_id'), terminate=True) delete_export_status() @@ -178,6 +229,30 @@ def cancel(self): }) + # 再開用メソッド + @author_permission.require(http_exception=403) + @expose('/resume', methods=['POST']) + def resume(self): + """Resume export progress.""" + + delete_export_url() + temp_folder_path = current_app.config.get("WEKO_AUTHORS_EXPORT_TEMP_FOLDER_PATH") + os.makedirs(temp_folder_path, exist_ok=True) + prefix = current_app.config["WEKO_AUTHORS_EXPORT_TMP_PREFIX"] + datetime.datetime.now().strftime("%Y%m%d%H%M") + + with tempfile.NamedTemporaryFile(dir=temp_folder_path, prefix=prefix, suffix='.tsv', mode='w+', delete=False) as temp_file: + temp_file_path = temp_file.name + update_cache_data(current_app.config["WEKO_AUTHORS_EXPORT_CACHE_TEMP_FILE_PATH_KEY"], + temp_file_path, + current_app.config["WEKO_AUTHORS_CACHE_TTL"] + ) + task = export_all.delay("author_db") + set_export_status(task_id=task.id) + return jsonify({ + 'code': 200, + 'data': {'task_id': task.id} + }) + class ImportView(BaseView): """Weko import authors admin view.""" @@ -196,57 +271,201 @@ def is_import_available(self): 'group_task_id', type=str, default=None) return jsonify(check_is_import_available(group_task_id)) - + @author_permission.require(http_exception=403) @expose('/check_import_file', methods=['POST']) def check_import_file(self): """Validate author import.""" - error = None + error = None list_import_data = [] - json_data = request.get_json() + counts = 0 + max_page = 1 if json_data: - result = check_import_data( - json_data.get('file_name'), - json_data.get('file').split(",")[-1] - ) - error = result.get('error') - list_import_data = result.get('list_import_data') - + target = json_data.get('target') + if target == 'id_prefix' or target == 'affiliation_id': + result = check_import_data_for_prefix( + target, + json_data.get('file_name'), + json_data.get('file').split(",")[-1] + ) + list_import_data = result.get('list_import_data') + elif target == 'author_db': + temp_folder_path = current_app.config.get("WEKO_AUTHORS_IMPORT_TEMP_FOLDER_PATH") + os.makedirs(temp_folder_path, exist_ok=True) + prefix = current_app.config["WEKO_AUTHORS_IMPORT_TMP_PREFIX"] + datetime.datetime.now().strftime("%Y%m%d%H%M") + file_suffix = json_data.get('file_name').split('.')[-1].lower() + with tempfile.NamedTemporaryFile(dir=temp_folder_path, prefix=prefix, suffix='.'+file_suffix, mode='wb', delete=False) as temp_file: + temp_file_path = temp_file.name + temp_file.write(base64.b64decode(str(json_data.get('file').split(",")[-1]))) + temp_file.flush() + + update_cache_data(current_app.config["WEKO_AUTHORS_IMPORT_CACHE_USER_TSV_FILE_KEY"], + temp_file_path, + current_app.config["WEKO_AUTHORS_CACHE_TTL"] + ) + result = check_import_data(os.path.basename(temp_file_path)) + error = result.get('error') + list_import_data = result.get('list_import_data') + counts = result.get("counts") + max_page = result.get("max_page") + + # インポートタブのチェック結果一時ファイルがあれば削除 + band_file_path = current_cache.get(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_BAND_CHECK_USER_FILE_PATH_KEY"]) + if band_file_path: + try: + current_cache.delete(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_BAND_CHECK_USER_FILE_PATH_KEY"]) + os.remove(band_file_path) + current_app.logger.debug(f"Deleted: {band_file_path}") + except Exception as e: + current_app.logger.error(f"Error deleting {band_file_path}: {e}") return jsonify( code=1, error=error, - list_import_data=list_import_data) + list_import_data=list_import_data, + counts=counts, + max_page=max_page) + + @author_permission.require(http_exception=403) + @expose('/check_pagination', methods=['GET']) + def check_pagination(self) -> jsonify: + """pagination checkfile""" + page_number = request.args.get("page_number") + temp_file_path = current_cache.get(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_USER_TSV_FILE_KEY"]) + temp_folder_path = current_app.config.get("WEKO_AUTHORS_IMPORT_TEMP_FOLDER_PATH") + + # checkファイルパスの作成 + base_file_name = os.path.splitext(os.path.basename(temp_file_path))[0] + check_file_name = f"{base_file_name}-check" + part_check_file_name = f"{check_file_name}-part{page_number}" + check_file_part1_path = os.path.join(temp_folder_path, part_check_file_name) + + with open(check_file_part1_path, "r", encoding="utf-8-sig") as check_part_file: + result = json.load(check_part_file) + return jsonify(result) + + @author_permission.require(http_exception=403) + @expose('/check_file_download', methods=['POST']) + def check_file_download(self): + """インポートチェック画面でダウンロード処理""" + band_file_path = current_cache.get(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_BAND_CHECK_USER_FILE_PATH_KEY"]) + if not band_file_path: + max_page = request.get_json().get("max_page") + band_file_path = band_check_file_for_user(max_page) + try: + return send_file(band_file_path, as_attachment=True) + except Exception as e: + current_app.logger.error(e) + return jsonify(msg=_('Failed')), 500 @author_permission.require(http_exception=403) @expose('/import', methods=['POST']) def import_authors(self) -> jsonify: - """Import author into System.""" + """Import author or other info into System.""" data = request.get_json() or {} - + is_target = data.get("isTarget","") # check import feature is available before import result_check = check_is_import_available(data.get('group_task_id')) if not result_check['is_available']: return jsonify(result_check) - + force_change_mode = data.get("force_change_mode", False) + tasks = [] records = [item for item in data.get( 'records', []) if not item.get('errors')] - group_tasks = [] - for author in records: - group_tasks.append(import_author.s(author)) - + count=0 + + if is_target == "id_prefix": + for id_prefix in records: + group_tasks.append(import_id_prefix.s(id_prefix)) + elif is_target == "affiliation_id": + for affiliation_id in records: + group_tasks.append(import_affiliation_id.s(affiliation_id)) + elif is_target == "author_db": + # 既にある結果一時ファイルの削除 + result_over_max_file_path = current_cache.get(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_OVER_MAX_FILE_PATH_KEY"]) + result_file_path = current_cache.get(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_FILE_PATH_KEY"]) + + if result_over_max_file_path: + try: + current_cache.delete(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_OVER_MAX_FILE_PATH_KEY"]) + os.remove(result_over_max_file_path) + current_app.logger.debug(f"Deleted: {result_over_max_file_path}") + except Exception as e: + current_app.logger.error(f"Error deleting {result_over_max_file_path}: {e}") + if result_file_path: + try: + current_cache.delete(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_FILE_PATH_KEY"]) + os.remove(result_file_path) + current_app.logger.debug(f"Deleted: {result_file_path}") + except Exception as e: + current_app.logger.error(f"Error deleting {result_file_path}: {e}") + + # 前回のimport結果サマリーを削除 + result_summary = current_cache.get(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY"]) + if result_summary: + current_cache.delete(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY"]) + + max_page_for_import_tab = data.get("max_page") + + # フロントの最大表示数分だけrecordsを確保 + records, reached_point, count = prepare_import_data(max_page_for_import_tab) + task_ids =[] + + for author in records: + group_tasks.append(import_author.s(author, force_change_mode)) + else: + return jsonify({'status': 'fail', 'message': 'Invalid target'}) + # handle import tasks import_task = group(group_tasks).apply_async() import_task.save() - for idx, task in enumerate(import_task.children): - tasks.append({ - 'task_id': task.task_id, - 'record_id': records[idx].get('pk_id'), - 'status': 'PENDING' - }) + + if is_target == "id_prefix" or is_target == "affiliation_id": + for idx, task in enumerate(import_task.children): + tasks.append({ + 'task_id': task.task_id, + 'scheme': records[idx].get('scheme'), + 'name': records[idx].get('name'), + 'status': 'PENDING' + }) + elif is_target == "author_db": + for idx, task in enumerate(import_task.children): + tasks.append({ + 'task_id': task.task_id, + 'record_id': records[idx].get('pk_id'), + 'previous_weko_id': records[idx].get('current_weko_id'), + 'new_weko_id': records[idx].get('weko_id'), + 'status': 'PENDING' + }) + task_ids.append(task.task_id) + + # WEKO_AUTHORS_IMPORT_MAX_NUM_OF_DISPLAYSを超えた分を別のタスクで処理 + if count > current_app.config.get("WEKO_AUTHORS_IMPORT_MAX_NUM_OF_DISPLAYS"): + update_cache_data(\ + current_app.config.get("WEKO_AUTHORS_IMPORT_CACHE_FORCE_CHANGE_MODE_KEY"), + force_change_mode, + current_app.config.get("WEKO_AUTHORS_CACHE_TTL") + ) + task = import_author_over_max.delay(reached_point ,task_ids, max_page_for_import_tab) + update_cache_data(\ + current_app.config.get("WEKO_AUTHORS_IMPORT_CACHE_OVER_MAX_TASK_KEY"), + task.id, + current_app.config.get("WEKO_AUTHORS_CACHE_TTL") + ) + response_data = { 'group_task_id': import_task.id, "tasks": tasks @@ -256,29 +475,51 @@ def import_authors(self) -> jsonify: {**response_data, **{'records': records}}, 0 ) - + response_object = { "status": "success", - "data": {**response_data} + "count": count, + "data": {**response_data}, + "records":records } return jsonify(response_object) + + def get_task(self, target, task_id): + tasks = { + "id_prefix": import_id_prefix.AsyncResult, + "affiliation_id": import_affiliation_id.AsyncResult, + "author_db": import_author.AsyncResult + } + return tasks.get(target, lambda x: None)(task_id) @expose('/check_import_status', methods=['POST']) def check_import_status(self): """Is import available.""" + result_task={} result = [] data = request.get_json() or {} + success_count = 0 + failure_count = 0 + + target = data.get("isTarget","") if data and data.get('tasks'): for task_id in data.get('tasks'): - task = import_author.AsyncResult(task_id) - start_date = task.result['start_date'] if task.result else '' - end_date = task.result['end_date'] if task.result else '' + task = self.get_task(target, task_id) + start_date = "" + end_date = "" + if task.result: + start_date = task.result.get('start_date', '') + end_date = task.result.get('end_date', '') status = states.PENDING error_id = None if task.result and task.result.get('status'): status = task.result.get('status') error_id = task.result.get('error_id') + if status == states.SUCCESS: + success_count += 1 + elif status == states.FAILURE: + failure_count += 1 result.append({ "task_id": task_id, "start_date": start_date, @@ -286,10 +527,54 @@ def check_import_status(self): "status": status, "error_id": error_id }) + if target == "author_db": + over_max_task = current_cache.get(\ + current_app.config.get("WEKO_AUTHORS_IMPORT_CACHE_OVER_MAX_TASK_KEY") + ) + if over_max_task: + task = import_author_over_max.AsyncResult(over_max_task) + status = states.PENDING + error_id = None + if task.result and task.result.get('status'): + status = task.result.get('status') + error_id = task.result.get('error_id') + result_task["over_max"]={ + "task_id": over_max_task, + "status": status, + "error_id": error_id + } + summary = current_cache.get(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY"]) + if summary: + success_count += summary["success_count"] + failure_count += summary["failure_count"] + result_task["summary"] = { + "success_count": success_count, + "failure_count": failure_count + } + result_task["tasks"] = result + else: + result_task = result + return jsonify(result_task) - return jsonify(result) - - + @author_permission.require(http_exception=403) + @expose('/result_download', methods=['POST']) + def result_file_download(self): + """インポート結果画面でダウンロード処理""" + json = request.get_json().get("json") + result_file_path = current_cache.get(\ + current_app.config["WEKO_AUTHORS_IMPORT_CACHE_RESULT_FILE_PATH_KEY"]) + if not result_file_path: + result_file_path = create_result_file_for_user(json) + if not result_file_path: + return jsonify({"Result": "Dont need to create result file"}) + try: + return send_file(result_file_path, as_attachment=True) + except Exception as e: + current_app.logger.error(e) + return jsonify(msg=_('Failed')), 500 + + authors_list_adminview = { 'view_class': AuthorManagementView, 'kwargs': { diff --git a/modules/weko-authors/weko_authors/api.py b/modules/weko-authors/weko_authors/api.py index 5ca045c828..fd3db32f41 100644 --- a/modules/weko-authors/weko_authors/api.py +++ b/modules/weko-authors/weko_authors/api.py @@ -24,11 +24,13 @@ import uuid from flask import current_app, json +from flask_security import current_user from invenio_db import db from invenio_indexer.api import RecordIndexer from sqlalchemy.sql.functions import func +from sqlalchemy.exc import SQLAlchemyError +from time import sleep -from weko_authors.config import WEKO_AUTHORS_FILE_MAPPING from .models import Authors, AuthorsPrefixSettings, AuthorsAffiliationSettings @@ -46,14 +48,6 @@ def create(cls, data): new_id = Authors.get_sequence(session) data["pk_id"] = str(new_id) data["gather_flg"] = 0 - data["authorIdInfo"].insert( - 0, - { - "idType": "1", - "authorId": str(new_id), - "authorIdShowFlg": "true" - } - ) es_id = str(uuid.uuid4()) es_data = json.loads(json.dumps(data)) @@ -63,6 +57,7 @@ def create(cls, data): author = Authors(id=new_id, json=data) session.add(author) except Exception as ex: + session.rollback() raise ex else: RecordIndexer().client.index( @@ -73,7 +68,7 @@ def create(cls, data): ) @classmethod - def update(cls, author_id, data): + def update(cls, author_id, data, force_change=False): """Update author.""" def update_es_data(data, es_id): """Update author data in ES.""" @@ -128,9 +123,18 @@ def update_es_data(data, es_id): author.json = data db.session.merge(author) except Exception as ex: + db.session.rollback() raise ex else: update_es_data(es_data, es_id) + from weko_deposit.tasks import update_items_by_authorInfo + if not current_user: + user_id = None + else: + user_id = current_user.get_id() + + update_items_by_authorInfo.delay( + user_id, data, [author_id], [data["id"]], force_change=force_change) @classmethod def get_all(cls, with_deleted=True, with_gather=True): @@ -143,21 +147,60 @@ def get_all(cls, with_deleted=True, with_gather=True): filters.append(Authors.gather_flg == 0) query = Authors.query.filter(*filters) query = query.order_by(Authors.id) + query.count() return query.all() + @classmethod + def get_by_range(cls, start_point, sum, with_deleted=True, with_gather=True): + """Get authors by range.""" + filters = [] + try: + with db.session.no_autoflush: + if not with_deleted: + filters.append(Authors.is_deleted.is_(False)) + if not with_gather: + filters.append(Authors.gather_flg == 0) + query = Authors.query.filter(*filters) + query = query.order_by(Authors.id) + query = query.offset(start_point).limit(sum) + return query.all() + except Exception as ex: + current_app.logger.error(ex) + raise ex + + @classmethod + def get_records_count(cls, with_deleted=True, with_gather=True): + """Get authors's count.""" + filters = [] + try: + with db.session.no_autoflush: + if not with_deleted: + filters.append(Authors.is_deleted.is_(False)) + if not with_gather: + filters.append(Authors.gather_flg == 0) + query = Authors.query.filter(*filters) + query = query.order_by(Authors.id) + return query.count() + except Exception as ex: + current_app.logger.error(ex) + raise ex + + @classmethod def get_author_for_validation(cls): """Get new author id.""" existed_authors_id = {} existed_external_authors_id = {} for author in cls.get_all(): - existed_authors_id[str(author.id)] = not author.is_deleted \ - and author.gather_flg == 0 + not_deleted_and_gather = not author.is_deleted and author.gather_flg == 0 + existed_authors_id[str(author.id)] = not_deleted_and_gather metadata = author.json + if not not_deleted_and_gather: + continue for authorIdInfo in metadata.get('authorIdInfo', {}): idType = authorIdInfo.get('idType') - if idType and idType != '1': + if idType: author_ids = existed_external_authors_id.get(idType, {}) weko_ids = author_ids.get(authorIdInfo.get('authorId'), []) weko_ids.append(str(author.id)) @@ -166,6 +209,125 @@ def get_author_for_validation(cls): return existed_authors_id, existed_external_authors_id + @classmethod + def get_pk_id_by_weko_id(cls, weko_id): + """ + Get pk_id by weko_id. + """ + query = { + "_source": ["pk_id", "authorIdInfo"], + "query": { + "bool": { + "must": [ + {"term": {"authorIdInfo.authorId": weko_id}}, + {"term": {"gather_flg": {"value": 0}}} + ], + "must_not": [ + {"term": {"is_deleted": True}} + ] + } + } + } + + # 検索 + indexer = RecordIndexer() + result = indexer.client.search( + index=current_app.config['WEKO_AUTHORS_ES_INDEX_NAME'], + body=query + ) + + for res in result['hits']['hits']: + author_id_info_from_es = res['_source']['authorIdInfo'] + for info in author_id_info_from_es: + if info.get('idType') == '1': + author_id = info.get('authorId') + if author_id == weko_id: + pk_id = res['_source']['pk_id'] + return pk_id + return -1 + + @classmethod + def get_weko_id_by_pk_id(cls, pk_id): + """pk_idからweko_idを取得します。 + + Args: + pk_id (str): pk_id + + Returns: + weko_id :str + """ + try: + with db.session.begin_nested(): + author = Authors.query.filter_by(id=pk_id).one_or_none() + if not author: + return None + json = author.json + for author_id_info in json["authorIdInfo"]: + if author_id_info["idType"] == "1": + weko_id = author_id_info["authorId"] + break + return weko_id + except Exception as ex: + raise ex + + @classmethod + def get_used_scheme_of_id_prefix(cls): + """get used scheme of id prefix.""" + id_prefixes = cls.get_id_prefix_all() + idtype_and_scheme ={} + if id_prefixes: + for id_prefix in id_prefixes: + idtype_and_scheme[id_prefix.id] = id_prefix.scheme + used_external_id_prefix = [] + for author in cls.get_all(): + metadata = author.json + for authorIdInfo in metadata.get('authorIdInfo', {}): + idType = authorIdInfo.get('idType') + if idType and idType != '1' \ + and idtype_and_scheme.get(int(idType)) not in used_external_id_prefix: + used_external_id_prefix.append(idtype_and_scheme.get(int(idType))) + return used_external_id_prefix, idtype_and_scheme + + @classmethod + def get_used_scheme_of_affiliation_id(cls): + """get used scheme of affiliation id.""" + affiliaiton_ids = cls.get_affiliation_id_all() + idtype_and_scheme ={} + if affiliaiton_ids: + for affiliaiton_id in affiliaiton_ids: + idtype_and_scheme[affiliaiton_id.id] = affiliaiton_id.scheme + used_external_id = [] + for author in cls.get_all(): + metadata = author.json + for affiliationInfo in metadata.get('affiliationInfo', []): + for identifierInfo in affiliationInfo.get('identifierInfo', []): + idType = identifierInfo.get('affiliationIdType') + if idType \ + and idtype_and_scheme.get(int(idType)) not in used_external_id: + used_external_id.append(idtype_and_scheme.get(int(idType))) + return used_external_id, idtype_and_scheme + + + + @classmethod + def get_id_prefix_all(cls): + """Get all id_prefix.""" + with db.session.no_autoflush: + query = AuthorsPrefixSettings.query + query = query.order_by(AuthorsPrefixSettings.id) + + return query.all() + + @classmethod + def get_scheme_of_id_prefix(cls): + """Get all _id_prefix scheme.""" + result = [] + id_prefixes = cls.get_id_prefix_all() + if id_prefixes: + for id_prefix in id_prefixes: + result.append(id_prefix.scheme) + return result + @classmethod def get_identifier_scheme_info(cls): """Get all Author Identifier Scheme informations.""" @@ -177,39 +339,147 @@ def get_identifier_scheme_info(cls): for scheme in schemes: result[str(scheme.id)] = dict( scheme=scheme.scheme, url=scheme.url) + return result + + @classmethod + def get_affiliation_id_all(cls): + """Get all affiliation_id.""" + with db.session.no_autoflush: + query = AuthorsAffiliationSettings.query + query = query.order_by(AuthorsAffiliationSettings.id) + + return query.all() + + @classmethod + def get_scheme_of_affiliaiton_id(cls): + """Get all affiliation_id scheme.""" + result = [] + affiliaiton_ids = cls.get_affiliation_id_all() + if affiliaiton_ids: + for affiliation_id in affiliaiton_ids: + result.append(affiliation_id.scheme) + return result + @classmethod + def get_affiliation_identifier_scheme_info(cls): + result = {} + with db.session.no_autoflush: + schemes = AuthorsAffiliationSettings.query.order_by( + AuthorsAffiliationSettings.id).all() + if schemes: + for scheme in schemes: + result[str(scheme.id)] = dict( + scheme=scheme.scheme, url=scheme.url) return result @classmethod - def prepare_export_data(cls, mappings, authors, schemes): + def mapping_max_item(cls, mappings, affiliation_mappings, records_count, retrys=0): + """Mapping max item of multiple case.""" + try: + size = current_app.config["WEKO_AUTHORS_EXPORT_BATCH_SIZE"] + if not mappings: + mappings = deepcopy(current_app.config["WEKO_AUTHORS_FILE_MAPPING"]) + if not affiliation_mappings: + affiliation_mappings = deepcopy(current_app.config["WEKO_AUTHORS_FILE_MAPPING_FOR_AFFILIATION"]) + if not records_count: + records_count = cls.get_records_count(False, False) + # 削除されておらず、統合されていない著者の合計を取得 + affiliation_mappings["max"] = [] + + # AUTHOR_EXPORT_BATCH_SIZE分の数だけauthorを取得して、maxを計算する + for i in range(0,records_count-1, size): + authors = cls.get_by_range(i, size, False, False) + # 通常のマッピングの処理 + for mapping in mappings: + if mapping.get('child'): + if not authors: + mapping['max'] = max(mapping.get('max', 1), 1) + else: + mapping['max'] = max( + mapping.get('max', 1), + max( + list(map(lambda x: len(x.json.get(mapping['json_id'], [])), authors)) + ) + ) + if mapping['max'] == 0: + mapping['max'] = 1 + + # 所属情報のマッピングの処理 + mapping_max = affiliation_mappings["max"] + # 著者DBの所属情報の最大値をそれぞれとる + for author in authors: + json_data = author.json + # affiliation_mappings配下に以下を作る。 + # affiliationInfoの列数だけこれが作られるものとする。 + # max:[ + # { + # "identifierInfo" : affiliationInfo[i]identifierInfoの長さ, + # "affiliationNameInfo" : affiliationInfo[i]affiliationNameInfoの長さ, + # "affiliationPeriodInfo" : affiliationInfo[i]affiliationPeriodInfoの長さ + # }, + # ] + for index, affiliation in enumerate(json_data.get(affiliation_mappings["json_id"], [])): + if len(mapping_max) < index+1 : + mapping_max.append({ + "identifierInfo" : 1, + "affiliationNameInfo" : 1, + "affiliationPeriodInfo" : 1, + }) + for child in affiliation_mappings["child"]: + child_length = len(affiliation.get(child["json_id"], [])) + if child_length > mapping_max[index][child["json_id"]]: + mapping_max[index][child["json_id"]] = child_length + + # 最後にWEKOIDの分を最大値から引く + for mapping in mappings: + if mapping['json_id'] == 'authorIdInfo': + if mapping['max'] > 1: + mapping['max'] -= 1 + + except SQLAlchemyError as ex: + current_app.logger.error(ex) + _num_retry = current_app.config["WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY"] + sleep_time = current_app.config["WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL"] + if retrys < _num_retry: + retrys += 1 + current_app.logger.info("mapping_max_item retry count: {}".format(retrys)) + db.session.rollback() + sleep(sleep_time) + result = cls.mapping_max_item( + mappings=None, affiliation_mappings=None, \ + records_count=records_count, retrys=retrys + ) + return result + else: + raise ex + except Exception as ex: + current_app.logger.error(ex) + raise ex + return mappings, affiliation_mappings + + @classmethod + def prepare_export_data(cls, mappings, affiliation_mappings, authors, schemes, aff_schemes, start, size): """Prepare export data of all authors.""" row_header = [] row_label_en = [] row_label_jp = [] row_data = [] - - if not mappings: - mappings = deepcopy(WEKO_AUTHORS_FILE_MAPPING) + + if not mappings or not affiliation_mappings: + mappings, affiliation_mappings = WekoAuthors.mapping_max_item(\ + deepcopy(current_app.config["WEKO_AUTHORS_FILE_MAPPING"]), + deepcopy(current_app.config["WEKO_AUTHORS_FILE_MAPPING_FOR_AFFILIATION"]), + WekoAuthors.get_records_count(False, False) + ) if not authors: - authors = cls.get_all(with_deleted=False, with_gather=False) + authors = cls.get_by_range(start, size, False, False) if not schemes: schemes = cls.get_identifier_scheme_info() - + if not aff_schemes: + aff_schemes = cls.get_affiliation_identifier_scheme_info() # calc max item of multiple case and prepare header, label for mapping in mappings: if mapping.get('child'): - if not authors: - mapping['max'] = 1 - else: - mapping['max'] = max( - list(map(lambda x: len(x.json.get(mapping['json_id'], [])), authors)) - ) - if mapping['max'] == 0: - mapping['max'] = 1 - if authors and mapping['json_id'] == 'authorIdInfo': - if mapping['max'] > 1: - mapping['max'] -= 1 - for i in range(0, mapping['max']): for child in mapping.get('child'): row_header.append('{}[{}].{}'.format( @@ -222,6 +492,23 @@ def prepare_export_data(cls, mappings, authors, schemes): row_header.append(mapping['json_id']) row_label_en.append(mapping['label_en']) row_label_jp.append(mapping['label_jp']) + + # 所属情報のマッピングの処理 + aff_mapping_max = affiliation_mappings["max"] + # tsv用に修正 + for index, m_max in enumerate(aff_mapping_max): + for child in affiliation_mappings.get('child'): + child_json_id = child["json_id"] + for i in range(m_max[child_json_id]): + for c in child["child"]: + # for child in mapping.get: + row_header.append('{}[{}].{}[{}].{}'.format( + affiliation_mappings["json_id"], index, child_json_id, i, c['json_id'])) + row_label_en.append( + '{}[{}][{}]'.format(c['label_en'], index, i)) + row_label_jp.append( + '{}[{}][{}]'.format(c['label_jp'], index, i)) + row_header[0] = '#' + row_header[0] row_label_en[0] = '#' + row_label_en[0] row_label_jp[0] = '#' + row_label_jp[0] @@ -232,7 +519,7 @@ def prepare_export_data(cls, mappings, authors, schemes): row = [] for mapping in mappings: if mapping.get('child'): - data = json_data.get(mapping['json_id']) + data = json_data.get(mapping['json_id'], []) idx_start = 0 idx_size = mapping['max'] # ignore WEKO id @@ -255,7 +542,7 @@ def prepare_export_data(cls, mappings, authors, schemes): else: val = data[i].get(child['json_id']) if child['json_id'] == 'idType': - scheme = schemes.get(val) + scheme = schemes.get(str(val)) row.append( scheme['scheme'] if scheme else val) else: @@ -269,9 +556,65 @@ def prepare_export_data(cls, mappings, authors, schemes): None ) ) + elif mapping["json_id"] == "weko_id": + id_info = json_data["authorIdInfo"][0] + if id_info["idType"] == "1": + row.append(id_info["authorId"]) + else: + row.append(None) else: row.append(json_data.get(mapping['json_id'])) + # 各affilitionInfo + aff_data = json_data.get(affiliation_mappings['json_id'], []) + affiliation_idx_start = 0 + affiliation_idx_size = len(aff_mapping_max) + # affilaitionのマックスだけ繰り返す + for i in range(affiliation_idx_start, affiliation_idx_size): + for child in affiliation_mappings.get('child'): + aff_child_idx_start = 0 + aff_child_idx_size = aff_mapping_max[i][child["json_id"]] + for j in range(aff_child_idx_start, aff_child_idx_size): + for c in child.get("child"): + if i >= len(aff_data): + row.append(None) + continue + aff_d = aff_data[i].get(child["json_id"], []) + if j >= len(aff_d): + row.append(None) + continue + if "mask" in c: + row.append( + c['mask'].get( + str(aff_d[j].get( + c["json_id"])).lower(), + None + ) + ) + else: + val = aff_d[j].get(c["json_id"]) + if c["json_id"] == "affiliationIdType": + aff_scheme = aff_schemes.get(val) + row.append( + aff_scheme["scheme"] if aff_scheme else val + ) + else: + row.append(val) + row_data.append(row) return row_header, row_label_en, row_label_jp, row_data + + @classmethod + def prepare_export_prefix(cls, target_prefix, prefixes): + """Prepare export data of id_prefix, affiliation_id.""" + row_data = [] + + for prefix in prefixes: + # 著者識別子のプレフィックスはWEKOのものは除外 + if target_prefix == "id_prefix" and prefix.scheme == "WEKO": + continue + row = [prefix.scheme, prefix.name, prefix.url] + row_data.append(row) + + return row_data \ No newline at end of file diff --git a/modules/weko-authors/weko_authors/config.py b/modules/weko-authors/weko_authors/config.py index e61cda7746..fa9db8daf9 100644 --- a/modules/weko-authors/weko_authors/config.py +++ b/modules/weko-authors/weko_authors/config.py @@ -19,8 +19,9 @@ # MA 02111-1307, USA. """Configuration for weko-authors.""" - +import os from invenio_stats.config import SEARCH_INDEX_PREFIX as index_prefix +from .rest import Authors WEKO_AUTHORS_LIST_SCHEME = ['e-Rad', 'NRID', 'ORCID', 'ISNI', 'VIAF', 'AID', 'kakenhi', 'Ringgold', 'GRID', 'ROR', 'Other'] @@ -58,19 +59,44 @@ WEKO_AUTHORS_EXPORT_ENTRYPOINTS = { 'export': '/admin/authors/export/export', 'cancel': '/admin/authors/export/cancel', - 'check_status': '/admin/authors/export/check_status' + 'check_status': '/admin/authors/export/check_status', + 'stop': '/admin/authors/export/stop', + 'resume': '/admin/authors/export/resume' } WEKO_AUTHORS_EXPORT_FILE_NAME = 'Creator_export_all' +WEKO_AUTHORS_ID_PREFIX_EXPORT_FILE_NAME = 'Id_prefix_export_all' +WEKO_AUTHORS_AFFILIATION_EXPORT_FILE_NAME = 'Affiliation_id_export_all' WEKO_AUTHORS_EXPORT_CACHE_STATUS_KEY = 'weko_authors_export_status' WEKO_AUTHORS_EXPORT_CACHE_URL_KEY = 'weko_authors_exported_url' +WEKO_AUTHORS_EXPORT_TARGET_CACHE_KEY = 'weko_authors_export_target' +WEKO_AUTHORS_EXPORT_CACHE_KEY = 'weko_author_export_cache_key' +WEKO_AUTHORS_EXPORT_CACHE_TEMP_FILE_PATH_KEY = 'weko_authors_export_temp_file_path_key' +WEKO_AUTHORS_EXPORT_CACHE_STOP_POINT_KEY = "weko_authors_export_stop_point" +WEKO_AUTHORS_EXPORT_TMP_PREFIX = 'authors_export_' +WEKO_AUTHORS_EXPORT_TEMP_FOLDER_PATH = "/var/tmp/authors_export" # os.path.join(os.environ.get("TMPDIR"), "authors_export") +WEKO_AUTHORS_EXPORT_BATCH_SIZE = 1000 +WEKO_AUTHORS_BULK_EXPORT_MAX_RETRY = 5 +WEKO_AUTHORS_BULK_EXPORT_RETRY_INTERVAL = 5 WEKO_AUTHORS_FILE_MAPPING = [ { - 'label_en': 'WEKO ID', - 'label_jp': 'WEKO ID', - 'json_id': 'pk_id' + "json_id": "pk_id", + "label_en": "Author ID", + "label_jp": "著者ID" + }, + { + "json_id": "weko_id", + "label_en": "WEKO ID", + "label_jp": "WEKO ID", + 'validation': { + 'validator': { + 'class_name': 'weko_authors.contrib.validation', + 'func_name': 'validate_digits_for_wekoid' + } + } }, + { 'json_id': 'authorNameInfo', 'child': [ @@ -187,19 +213,165 @@ } ] +WEKO_AUTHORS_FILE_MAPPING_FOR_PREFIX =["scheme", "name", "url", "is_deleted"] + +WEKO_AUTHORS_FILE_MAPPING_FOR_AFFILIATION ={ + "json_id": "affiliationInfo", + "child": [ + { + "json_id": "identifierInfo", + "child": [ + { + "json_id": "affiliationIdType", + "label_en": "Affiliation Identifier Scheme", + "label_jp": "外部所属機関ID 識別子", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_affiliation_identifier_scheme" + }, + "required": { + "if": [ + "affiliationId" + ] + } + } + }, + { + "json_id": "affiliationId", + "label_en": "Affiliation Identifier", + "label_jp": "外部所属機関ID", + "validation": { + "required": { + "if": [ + "affiliationIdType" + ] + } + } + }, + { + "json_id": "identifierShowFlg", + "label_en": "Affiliation Identifier Display", + "label_jp": "外部所属機関ID 表示／非表示", + "mask": { + "true": "Y", + "false": "N" + } + } + ] + }, + { + "json_id": "affiliationNameInfo", + "child": [ + { + "json_id": "affiliationName", + "label_en": "Affiliation Name", + "label_jp": "外部所属機関名" + }, + { + "json_id": "affiliationNameLang", + "label_en": "Language", + "label_jp": "言語", + "validation": { + "map": [ + "ja", + "ja-Kana", + "en", + "fr", + "it", + "de", + "es", + "zh-cn", + "zh-tw", + "ru", + "la", + "ms", + "eo", + "ar", + "el", + "ko" + ] + } + }, + { + "json_id": "affiliationNameShowFlg", + "label_en": "Affiliation Name Display", + "label_jp": "外部所属機関名・言語 表示／非表示", + "mask": { + "true": "Y", + "false": "N" + }, + "validation": { + "map": [ + "Y", + "N" + ] + } + } + ] + }, + { + "json_id": "affiliationPeriodInfo", + "child": [ + { + "json_id": "periodStart", + "label_en": "Affiliation Period Start", + "label_jp": "外部所属機関 所属期間 開始日", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_affiliation_period_start" + } + } + }, + { + "json_id": "periodEnd", + "label_en": "Affiliation Period End", + "label_jp": "外部所属機関 所属期間 終了日", + "validation": { + "validator": { + "class_name": "weko_authors.contrib.validation", + "func_name": "validate_affiliation_period_end" + } + } + } + ] + } + ] + } + + WEKO_AUTHORS_ADMIN_IMPORT_TEMPLATE = 'weko_authors/admin/author_import.html' """Template for the import page.""" -WEKO_AUTHORS_IMPORT_TMP_PREFIX = 'weko_authors_import_' +WEKO_AUTHORS_IMPORT_TMP_PREFIX = 'authors_import_' +WEKO_AUTHORS_IMPORT_TEMP_FOLDER_PATH = "var/tmp/authors_import"# os.path.join(os.environ.get("TMPDIR"), "authors_import") WEKO_AUTHORS_IMPORT_ENTRYPOINTS = { 'is_import_available': '/admin/authors/import/is_import_available', 'check_import_file': '/admin/authors/import/check_import_file', 'import': '/admin/authors/import/import', - 'check_import_status': '/admin/authors/import/check_import_status' + 'check_import_status': '/admin/authors/import/check_import_status', + 'check_pagination':'/admin/authors/import/check_pagination', + 'check_file_download':'/admin/authors/import/check_file_download', + 'result_download':'/admin/authors/import/result_download', } WEKO_AUTHORS_IMPORT_CACHE_KEY = 'author_import_cache' +WEKO_AUTHORS_IMPORT_CACHE_USER_TSV_FILE_KEY = 'authors_import_user_file_key' +WEKO_AUTHORS_IMPORT_CACHE_BAND_CHECK_USER_FILE_PATH_KEY = "authors_import_band_check_user_file_path" +WEKO_AUTHORS_IMPORT_CACHE_BAND_CHECK_FILE_PATH_KEY = "authors_import_band_check_file_path" +WEKO_AUTHORS_IMPORT_CACHE_RESULT_OVER_MAX_FILE_PATH_KEY = "authors_import_result_file_of_over_path" +WEKO_AUTHORS_IMPORT_CACHE_RESULT_FILE_PATH_KEY = "authors_import_result_file_path" +WEKO_AUTHORS_IMPORT_CACHE_OVER_MAX_TASK_KEY = "authors_import_over_max_task" +WEKO_AUTHORS_IMPORT_CACHE_RESULT_SUMMARY_KEY = "authors_import_summary" +WEKO_AUTHORS_IMPORT_CACHE_FORCE_CHANGE_MODE_KEY = "authors_import_force_change" +WEKO_AUTHORS_IMPORT_BATCH_SIZE = 100 +WEKO_AUTHORS_IMPORT_MAX_NUM_OF_DISPLAYS = 1000 +WEKO_AUTHORS_BULK_IMPORT_MAX_RETRY = 5 +WEKO_AUTHORS_BULK_IMPORT_RETRY_INTERVAL = 5 +WEKO_AUTHORS_CACHE_TTL = 60 * 60 * 24 +WEKO_AUTHORS_IMPORT_TEMP_FILE_RETENTION_PERIOD = 60 * 60 * 24 WEKO_AUTHORS_NUM_OF_PAGE = 25 """Default number of author search results that display in one page.""" @@ -305,3 +477,19 @@ "reg": string pattern, excluding first and last slashes from literal notation """ +WEKO_AUTHORS_REST_ENDPOINTS = { + 'authors': { + 'route': '//authors/count', + 'api_route': '//authors', + 'default_media_type': 'application/json', + 'record_serializers': { + 'application/json': ('invenio_records_rest.serializers' + ':json_v1_response'), + }, + }, +} + +WEKO_AUTHORS_COUNT_API_VERSION = { + 'get-v1': Authors.get_v1 +} +"""API version.""" \ No newline at end of file diff --git a/modules/weko-authors/weko_authors/contrib/validation.py b/modules/weko-authors/weko_authors/contrib/validation.py index 808c815534..7fa961c51c 100644 --- a/modules/weko-authors/weko_authors/contrib/validation.py +++ b/modules/weko-authors/weko_authors/contrib/validation.py @@ -21,16 +21,18 @@ """Validation for import weko-authors.""" import importlib +import re +from datetime import datetime from functools import reduce from operator import getitem from flask_babelex import gettext as _ from invenio_db import db -from weko_authors.models import AuthorsPrefixSettings +from weko_authors.models import AuthorsPrefixSettings, AuthorsAffiliationSettings -def validate_by_extend_validator(values=[], validator={}): +def validate_by_extend_validator(item, values=[], validator={}): """Validate by specify validator. Args: @@ -45,7 +47,7 @@ def validate_by_extend_validator(values=[], validator={}): if values and validator: _class = importlib.import_module(validator.get('class_name')) _func = getattr(_class, validator.get('func_name')) - errors = _func(values) + errors = _func(item, values) return errors @@ -91,8 +93,33 @@ def validate_map(values=[], _map=[]): errors_key.append(val['key']) return errors_key +def validate_digits_for_wekoid(items, values=[]): + """ + weko_idについてのバリデーションチェックします。 + 以下の場合にエラーを追加します。 + ・weko_idが存在しない場合 + ・weko_idが半角数字でない場合 + + Args: + item (dict): インポートされるデータをdictに直したもの、ここでは使わない + values (list, optional): List values with key path. Defaults to []. + + Returns: + list: List errors message. + """ + errors = [] + err_msg_format = _("WEKO ID is Half-width digits only") + err_msg = _("WEKO ID is required item.") + for val in values: + weko_id = val["value"] + if weko_id: + if not bool(re.fullmatch(r'[0-9]+', weko_id)): + errors.append(err_msg_format) + else: + errors.append(err_msg) + return errors -def validate_identifier_scheme(values=[]): +def validate_identifier_scheme(item, values=[]): """Validate Identifier Scheme. Args: @@ -113,6 +140,90 @@ def validate_identifier_scheme(values=[]): errors.append(err_mgs.format(val['value'])) return errors +def validate_affiliation_identifier_scheme(item, values=[]): + """Validate Affiliation Identifier Scheme. + + Args: + values (list, optional): List values with key path. Defaults to []. + + Returns: + list: List errors message. + + """ + errors = [] + err_msg = _("Specified Affiliation Identifier Scheme '{}' does not exist.") + if values: + with db.session.no_autoflush: + authors_prefix = AuthorsAffiliationSettings.query.all() + schemes = [prefix.scheme for prefix in authors_prefix] + for val in values: + if val['value'] and val['value'] not in schemes: + errors.append(err_msg.format(val['value'])) + return errors + + +def validate_affiliation_period_start(item, values=[]): + """ + periodStartをバリデーションする + チェックするのは以下 + ・yyyy-mm-ddの形であるか + + Args: + item (dict): インポートされるデータをdictに直したもの、ここでは使わない + values (list, optional): List values with key path. Defaults to []. + + Returns: + list: List errors message. + """ + errors = [] + err_msg_format = _("External Affiliation Period must be in the format: yyyy-MM-dd, blank. {}") + for val in values: + date = val["value"] + if date: + try: + datetime.strptime(date, "%Y-%m-%d") + except ValueError: + errors.append(err_msg_format.format(val['value'])) + return errors + +def validate_affiliation_period_end(item, values=[]): + """ + periodEndをバリデーションする + チェックするのは以下 + ・yyyy-mm-ddの形であるか + ・対になるperiodStartが正しいフォーマットの時、periodStartがperiodEndより遅くないか + + Args: + item (dict): インポートされるデータをdictに直したもの + values (list, optional): List values with key path. Defaults to []. + + Returns: + list: List errors message. + """ + errors = [] + err_msg_format = _("External Affiliation Period must be in the format: yyyy-MM-dd, blank. {}") + err_msg = _("Period end must be after Period start.") + for val in values: + period_end = val["value"] + if period_end: + try: + datetime.strptime(period_end, "%Y-%m-%d") + except ValueError: + errors.append(err_msg_format.format(val['value'])) + continue + reduce_keys = val['reduce_keys'] + uplevel_data = reduce(getitem, reduce_keys[:-1], item) + period_start = uplevel_data["periodStart"] + if period_start: + try: + period_start = datetime.strptime(period_start, "%Y-%m-%d") + period_end = datetime.strptime(period_end, "%Y-%m-%d") + except ValueError: + errors.append(err_msg_format.format(val['value'])) + continue + if period_start > period_end: + errors.append(err_msg) + return errors def validate_external_author_identifier(item, values=[], existed_external_authors_id={}): @@ -144,3 +255,24 @@ def validate_external_author_identifier(item, values=[], if warnings: return msg.format('
'.join(warnings)) return None + + +def check_weko_id_is_exits_for_import(pk_id, weko_id, existed_external_authors_id={}): + """weko_idがexisted_external_authors_idに存在するか確認します。 + 存在し、かつpk_idが一致しなかった場合、エラーを出します。 + + Args: + pk_id (str): pk_id + weko_id (str): weko_id + existed_external_authors_id (dict, optional): (object, optional): Existed external + author id. Defaults to {}. + + Returns: + list: List metadata paths are errors. + """ + errors = [] + err_msg = _("Specified WEKO ID already exist.") + exists_id = existed_external_authors_id["1"].get(weko_id) + if exists_id and not (pk_id in exists_id) : + errors.append(err_msg) + return errors \ No newline at end of file diff --git a/modules/weko-authors/weko_authors/errors.py b/modules/weko-authors/weko_authors/errors.py new file mode 100644 index 0000000000..66985d732a --- /dev/null +++ b/modules/weko-authors/weko_authors/errors.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +"""Authors Error""" +from invenio_rest.errors import RESTException + + +class VersionNotFoundRESTError(RESTException): + """API Version error.""" + + code = 400 + description = 'This API version does not found.' + + +class AuthorNotFoundRESTError(RESTException): + """Author Not Found error.""" + + code = 404 + description = 'This author does not found.' diff --git a/modules/weko-authors/weko_authors/ext.py b/modules/weko-authors/weko_authors/ext.py index 9244a111bf..4d8cc6bbc1 100644 --- a/modules/weko-authors/weko_authors/ext.py +++ b/modules/weko-authors/weko_authors/ext.py @@ -21,6 +21,7 @@ """Flask extension for weko-authors.""" from . import config +from .rest import create_blueprint class WekoAuthors(object): @@ -56,3 +57,39 @@ def init_config(self, app): for k in dir(config): if k.startswith('WEKO_AUTHORS_'): app.config.setdefault(k, getattr(config, k)) + + +class WekoAuthorsREST(object): + """weko-authors extension.""" + + def __init__(self, app=None): + """ + Extension initialization. + + :param app: An instance of :class:`flask.Flask`. + """ + if app: + self.init_app(app) + + def init_app(self, app): + """ + Flask application initialization. + Initialize the REST endpoints. + Connect all signals if `DEPOSIT_REGISTER_SIGNALS` is True. + + :param app: An instance of :class:`flask.Flask`. + """ + self.init_config(app) + blueprint = create_blueprint(app.config['WEKO_AUTHORS_REST_ENDPOINTS']) + app.register_blueprint(blueprint) + app.extensions['weko_authors_rest'] = self + + def init_config(self, app): + """ + Initialize configuration. + + :param app: An instance of :class:`flask.Flask`. + """ + for k in dir(config): + if k.startswith('WEKO_AUTHORS_'): + app.config.setdefault(k, getattr(config, k)) diff --git a/modules/weko-authors/weko_authors/models.py b/modules/weko-authors/weko_authors/models.py index 5b12580ea9..40e96db5bf 100644 --- a/modules/weko-authors/weko_authors/models.py +++ b/modules/weko-authors/weko_authors/models.py @@ -121,11 +121,12 @@ def get_author_by_id(cls, author_id): """ try: - author = cls.query.filter_by(id=author_id).one_or_none() - if not author: - return None - json_data = author.json - return json_data + with db.session.begin_nested(): + author = cls.query.filter_by(id=author_id).one_or_none() + if not author: + return None + json_data = author.json + return json_data except Exception: return None diff --git a/modules/weko-authors/weko_authors/rest.py b/modules/weko-authors/weko_authors/rest.py new file mode 100644 index 0000000000..15cc9bbb04 --- /dev/null +++ b/modules/weko-authors/weko_authors/rest.py @@ -0,0 +1,710 @@ +# -*- coding: utf-8 -*- +# +# This file is part of WEKO3. +# Copyright (C) 2017 National Institute of Informatics. +# +# WEKO3 is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# WEKO3 is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with WEKO3; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. + +"""Blueprint for schema rest.""" + +import json +import traceback +import inspect +import pdb + +from flask import Blueprint, current_app, Response, jsonify +from invenio_db import db +from invenio_rest import ContentNegotiatedMethodView +from .scopes import author_search_scope,author_create_scope,author_update_scope,author_delete_scope +from invenio_records_rest.utils import obj_or_import_string +from werkzeug.exceptions import InternalServerError, NotFound +from invenio_oauth2server import require_api_auth, require_oauth_scopes +from weko_accounts.utils import roles_required, limiter +from .errors import VersionNotFoundRESTError +from weko_authors.api import WekoAuthors +from weko_authors.utils import validate_weko_id, check_period_date + +from flask import request, jsonify, current_app +from sqlalchemy.exc import SQLAlchemyError +from werkzeug.exceptions import BadRequest +from invenio_search import current_search_client + +WEKO_ADMIN_PERMISSION_ROLE_SYSTEM = "System Administrator" +WEKO_ADMIN_PERMISSION_ROLE_REPO = "Repository Administrator" + +def create_blueprint(endpoints): + """Create Weko-Authors-REST blueprint. + + See: :data:`weko_authors.config.WEKO_AUTHORS_REST_ENDPOINTS`. + + :param endpoints: List of endpoints configuration. + :returns: The configured blueprint. + """ + blueprint = Blueprint( + 'weko_authors_rest', + __name__, + url_prefix='', + ) + + @blueprint.teardown_request + def dbsession_clean(exception): + current_app.logger.debug('weko_authors dbsession_clean: {}'.format(exception)) + if exception is None: + try: + db.session.commit() + except: + db.session.rollback() + db.session.remove() + + for endpoint, options in (endpoints or {}).items(): + record_serializers ={'application/json':jsonify} + if endpoint == 'authors': + authors = Authors.as_view( + Authors.view_name.format(endpoint), + default_media_type=options.get('default_media_type') + ) + author_api = AuthorDBManagementAPI.as_view( + AuthorDBManagementAPI.view_name.format(endpoint), + record_serializers=record_serializers, + default_media_type=options.get('default_media_type'), + ) + blueprint.add_url_rule( + options.get('route'), + view_func=authors, + methods=['GET'] + ) + blueprint.add_url_rule( + options.get('api_route'), + view_func=author_api, + methods=['GET','POST','PUT','DELETE'] + ) + + return blueprint + + +class Authors(ContentNegotiatedMethodView): + """Authors Resource.""" + view_name = '{0}' + + def __init__(self, *args, **kwargs): + """Constructor.""" + super(Authors, self).__init__( + *args, + **kwargs + ) + + def get(self, **kwargs): + """Count authors.""" + version = kwargs.get('version') + from .config import WEKO_AUTHORS_COUNT_API_VERSION + func = WEKO_AUTHORS_COUNT_API_VERSION.get(f'get-{version}') + + if func: + return func(self, **kwargs) + else: + raise VersionNotFoundRESTError() + + def get_v1(self, **kwargs): + + # Execute count. + try: + from .utils import count_authors + count_result = count_authors() + + # Create Response + res = Response( + response=json.dumps(count_result), + status=200, + content_type='application/json' + ) + + # Response header setting + res.headers['Cache-Control'] = 'no-store' + res.headers['Pragma'] = 'no-cache' + res.headers['Expires'] = 0 + + return res + + except Exception: + current_app.logger.error(traceback.print_exc()) + raise InternalServerError() + +class AuthorDBManagementAPI(ContentNegotiatedMethodView): + """Author Database Management API.""" + view_name = '{0}_db_management_API' + def __init__(self, record_serializers=None, + default_media_type=None, **kwargs): + """Constructor.""" + super(AuthorDBManagementAPI, self).__init__( + method_serializers={ + 'GET': record_serializers, + 'PUT': record_serializers, + 'POST': record_serializers, + 'DELETE': record_serializers, + }, + default_method_media_type={ + 'GET': default_media_type, + 'PUT': default_media_type, + 'POST': default_media_type, + 'DELETE': default_media_type, + }, + default_media_type=default_media_type, + **kwargs + ) + + @require_api_auth(allow_anonymous=False) + @require_oauth_scopes(author_search_scope.id) + @roles_required([WEKO_ADMIN_PERMISSION_ROLE_SYSTEM,WEKO_ADMIN_PERMISSION_ROLE_REPO]) + @limiter.limit('') + def get(self, **kwargs): + """Handle GET request.""" + version = kwargs.get('version') + func_name = f'get_{version}' + if hasattr(self, func_name): + return getattr(self, func_name)(**kwargs) + else: + raise VersionNotFoundRESTError() + + @require_api_auth(allow_anonymous=False) + @require_oauth_scopes(author_create_scope.id) + @roles_required([WEKO_ADMIN_PERMISSION_ROLE_SYSTEM,WEKO_ADMIN_PERMISSION_ROLE_REPO]) + @limiter.limit('') + def post(self, **kwargs): + """Handle GET request.""" + version = kwargs.get('version') + func_name = f'post_{version}' + if hasattr(self, func_name): + return getattr(self, func_name)(**kwargs) + else: + raise VersionNotFoundRESTError() + + @require_api_auth(allow_anonymous=False) + @require_oauth_scopes(author_update_scope.id) + @roles_required([WEKO_ADMIN_PERMISSION_ROLE_SYSTEM,WEKO_ADMIN_PERMISSION_ROLE_REPO]) + @limiter.limit('') + def put(self, **kwargs): + """Handle GET request.""" + version = kwargs.get('version') + func_name = f'put_{version}' + if hasattr(self, func_name): + return getattr(self, func_name)(**kwargs) + else: + raise VersionNotFoundRESTError() + + @require_api_auth(allow_anonymous=False) + @require_oauth_scopes(author_delete_scope.id) + @roles_required([WEKO_ADMIN_PERMISSION_ROLE_SYSTEM,WEKO_ADMIN_PERMISSION_ROLE_REPO]) + @limiter.limit('') + def delete(self, **kwargs): + """Handle GET request.""" + version = kwargs.get('version') + func_name = f'delete_{version}' + if hasattr(self, func_name): + return getattr(self, func_name)(**kwargs) + else: + raise VersionNotFoundRESTError() + + def get_v1(self, **kwargs): + """Handle GET request for v1.""" + from weko_authors.utils import get_author_prefix_obj,get_author_affiliation_obj,get_author_prefix_obj_by_id,get_author_affiliation_obj_by_id + try: + fullname = request.args.get("fullname") + firstname = request.args.get("firstname") + familyname = request.args.get("familyname") + idtype = request.args.get("idtype") + authorid = request.args.get("authorid") + + if (idtype and not authorid) or (authorid and not idtype): + raise BadRequest("Both 'idtype' and 'authorid' must be specified together or omitted.") + + # idtype (scheme -> id) + idtype_id = None + if idtype: + prefix_obj = get_author_prefix_obj(idtype) + if not prefix_obj: + raise BadRequest(f"Invalid idtype '{idtype}'.") + idtype_id = prefix_obj.id + + search_query = {"query": {"bool": {"must": []}}} + + if fullname: + parts = fullname.split(" ", 1) + firstname_ = parts[0] + familyname_ = parts[1] if len(parts) > 1 else None + + if firstname_ and familyname_: + search_query["query"]["bool"]["must"].append({"match": {"authorNameInfo.firstName": firstname_}}) + search_query["query"]["bool"]["must"].append({"match": {"authorNameInfo.familyName": familyname_}}) + + if firstname: + search_query["query"]["bool"]["must"].append({"match": {"authorNameInfo.firstName": firstname}}) + if familyname: + search_query["query"]["bool"]["must"].append({"match": {"authorNameInfo.familyName": familyname}}) + + if idtype and authorid: + search_query["query"]["bool"]["must"].append({ + "function_score": { + "query": { "match_all": {} }, + "script_score": { + "script": { + "lang": "painless", + "source": f"for (def author : params._source.authorIdInfo) {{ if (author.idType == '{idtype_id}' && author.authorId == '{authorid}') return 100; }} return 0;" + } + } + } + }) + search_query["min_score"] = 100 + + search_index = current_app.config.get("WEKO_AUTHORS_ES_INDEX_NAME") + search_results = current_search_client.search(index=search_index, body=search_query) + if search_results["hits"]["total"]> 0: + search_results = [author.get("_source", {}) for author in search_results.get("hits", {}).get("hits", [])] + else: + search_results = {} + + search_results = [self.remove_filed_no_need(author) for author in search_results] + search_results = [self.process_authors_data_after(author) for author in search_results] + + return Response( + response=json.dumps({"authors": search_results}), + status=200, + content_type='application/json' + ) + + except BadRequest as e: + raise BadRequest(str(e)) + except Exception as e: + current_app.logger.error(traceback.format_exc()) + raise InternalServerError("Internal server error.") + + def process_authors_data_before(self, author_data): + from weko_authors.utils import get_author_prefix_obj,get_author_affiliation_obj + # `authorIdInfo` (scheme -> id) + for auth_id in author_data.get("authorIdInfo", []): + prefix_obj = get_author_prefix_obj(auth_id.get("idType")) + if prefix_obj: + auth_id["idType"] = prefix_obj.id + # `affiliationIdType` (scheme -> id) + for affiliation in author_data.get("affiliationInfo", []): + for identifier in affiliation.get("identifierInfo", []): + aff_obj = get_author_affiliation_obj(identifier.get("affiliationIdType")) + if aff_obj: + identifier["affiliationIdType"] = aff_obj.id + return author_data + + def process_authors_data_after(self, author_data): + from weko_authors.utils import get_author_prefix_obj_by_id,get_author_affiliation_obj_by_id + # `authorIdInfo` (id -> scheme) + for auth_id in author_data.get("authorIdInfo", []): + prefix_obj = get_author_prefix_obj_by_id(auth_id.get("idType")) + if prefix_obj: + auth_id["idType"] = prefix_obj.scheme # 用 scheme 替换 idType + # `affiliationIdType` (id -> scheme) + for affiliation in author_data.get("affiliationInfo", []): + for identifier in affiliation.get("identifierInfo", []): + aff_obj = get_author_affiliation_obj_by_id(identifier.get("affiliationIdType")) + if aff_obj: + identifier["affiliationIdType"] = aff_obj.scheme + return author_data + + def remove_filed_no_need(self,data): + keys_to_keep = {"emailInfo", "authorIdInfo", "authorNameInfo", "affiliationInfo"} + return {k: v for k, v in data.items() if k in keys_to_keep} + + + def post_v1(self, **kwargs): + """Handle POST request for author registration.""" + import uuid + from weko_authors.models import Authors + try: + data = request.get_json() + + if not data: + raise BadRequest("Request body cannot be empty.") + author_data = data.get("author") + if not author_data: + raise BadRequest("author can not be null.") + + self.validate_author_data(author_data,author_data.get("pk_id")) + + author_data = self.process_authors_data_before(author_data) + + self.handle_weko_id(author_data) + + es_id = str(uuid.uuid4()) + author_data["id"] = es_id + + with db.session.begin_nested(): + new_id = Authors.get_sequence(db.session) + author_data["pk_id"] = str(new_id) + author_record = Authors(id=new_id, json=author_data, is_deleted=False) + db.session.add(author_record) + + author_data["gather_flg"] = 0 + author_data["is_deleted"] = False + author_data.pop("id") + + search_index = current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"] + current_search_client.index(index=search_index, id=es_id, body=author_data, doc_type="_doc") + return Response( + response=json.dumps({"message": "Author successfully registered.", "author": self.process_authors_data_after(author_data)}), + status=200, + content_type='application/json' + ) + + except BadRequest as e: + raise BadRequest(str(e)) + except SQLAlchemyError: + db.session.rollback() + raise InternalServerError("Database error.") + except Exception: + current_app.logger.error(traceback.format_exc()) + raise InternalServerError("Internal server error.") + + def validate_author_data(self, author_data, pk_id=None, is_update=False): + have_weko_id = False + for auth_id in author_data.get("authorIdInfo", []): + id_type = auth_id.get("idType") + author_id = auth_id.get("authorId") + + if bool(id_type) ^ bool(author_id): + raise BadRequest("Both 'idType' and 'authorId' must be provided together.") + + if id_type == "WEKO": + have_weko_id = True + is_valid, error_msg = validate_weko_id(author_id, pk_id) + if not is_valid and error_msg == "not half digit": + raise BadRequest("The WEKO ID must be numeric characters only.") + if not is_valid and error_msg == "already exists": + raise BadRequest("The value is already in use as WEKO ID.") + if not have_weko_id and is_update: + raise BadRequest(f"idType: WEKO (weko id) must be provided.") + + for name_info in author_data.get("authorNameInfo", []): + first_name = name_info.get("firstName") + family_name = name_info.get("familyName") + language = name_info.get("language") + + if (first_name or family_name) and not language: + raise BadRequest("If 'firstName' or 'familyName' is provided, 'language' must also be specified.") + + if language and first_name and family_name and not name_info.get("nameFormat"): + name_info["nameFormat"] = "familyNmAndNm" + + if not isinstance(author_data.get("affiliationInfo", []), list): + raise BadRequest("'affiliationInfo' should be a list.") + + for affiliation in author_data.get("affiliationInfo", []): + if not isinstance(affiliation, dict): + raise BadRequest("Invalid format in 'affiliationInfo'.") + + for identifier in affiliation.get("identifierInfo", []): + id_type = identifier.get("affiliationIdType") + aff_id = identifier.get("affiliationId") + + if bool(id_type) ^ bool(aff_id): + raise BadRequest("Both 'affiliationIdType' and 'affiliationId' must be provided together.") + + for name_info in affiliation.get("affiliationNameInfo", []): + aff_name = name_info.get("affiliationName") + aff_lang = name_info.get("affiliationNameLang") + + if bool(aff_name) ^ bool(aff_lang): + raise BadRequest("Both 'affiliationName' and 'affiliationNameLang' must be provided together.") + + is_valid, error_msg = check_period_date(author_data) + if not is_valid: + raise BadRequest(f"affiliationPeriodInfo error: {error_msg}") + + def handle_weko_id(self, author_data): + author_id_info = author_data.get("authorIdInfo", []) + has_weko_id = any(auth_id.get("idType") == "1" for auth_id in author_id_info) + + if not has_weko_id: + search_index = current_app.config.get("WEKO_AUTHORS_ES_INDEX_NAME") + + query = { + "size": 0, + "aggs": { + "max_weko_id": { + "max": { + "script": { + "source": """ + int maxId = 0; + for (def entry : params._source.authorIdInfo) { + if (entry.idType == '1') { + try { + int id = Integer.parseInt(entry.authorId); + if (id > maxId) maxId = id; + } catch (Exception e) { + // Ignore non-numeric values + } + } + } + return maxId; + """, + "lang": "painless" + } + } + } + } + } + + search_results = current_search_client.search(index=search_index, body=query) + max_weko_id = search_results["aggregations"]["max_weko_id"]["value"] or 0 + + new_weko_id = str(int(max_weko_id) + 1) + + author_id_info.append({ + "idType": "1", + "authorId": new_weko_id, + "authorIdShowFlg": "true" + }) + + author_data["authorIdInfo"] = author_id_info + + def put_v1(self, **kwargs): + """Handle PUT request for author update.""" + from weko_authors.models import Authors + from weko_deposit.tasks import update_items_by_authorInfo + from flask_login import current_user + + search_index = current_app.config["WEKO_AUTHORS_ES_INDEX_NAME"] + lang_options_list = [ + "ja", "ja-Kana", "en", "fr", "it", "de", "es", + "zh-cn", "zh-tw", "ru", "la", "ms", "eo", "ar", + "el", "ko" + ] + + try: + data = request.get_json() + + if not data: + raise BadRequest("Request body cannot be empty.") + author_data = data.get("author") + if not author_data: + raise BadRequest("author can not be null.") + + es_id = data.get("id") + pk_id = data.get("pk_id") + + if pk_id: + try: + pk_id = int(pk_id) + except ValueError: + raise BadRequest("Invalid author ID.") + + for data_filed in ["emailInfo","authorIdInfo","authorNameInfo","affiliationInfo"]: + if not author_data.get(data_filed,False): + print(f"[]{data_filed}") + author_data[data_filed] = [] + + prefix_schemes,affiliation_schemes = self.get_all_schemes() + if not self.validate_request_data(self.extract_data(author_data), lang_options_list, prefix_schemes, affiliation_schemes): + raise BadRequest("Invalid Author Data, 'idtype' or 'language' Not Allowed.") + + if not es_id and not pk_id: + raise BadRequest("Either 'id' or 'pk_id' must be specified.") + + author_by_pk = None + author_by_es = None + + if pk_id: + author_by_pk = Authors.query.filter_by(id=pk_id).first() + + if es_id: + search_results = current_search_client.search( + index=search_index, + body={"query": {"term": {"_id": es_id}}}, + size=1 + ) + if search_results["hits"]["total"] > 0: + author_by_es = search_results["hits"]["hits"][0]["_source"] + + if not author_by_pk and not author_by_es: + raise NotFound("Specified author does not exist.") + + if pk_id and es_id and (bool(author_by_pk) ^ bool(author_by_es)): + raise BadRequest("Parameters 'id' and 'pk_id' refer to different users.") + + + if author_by_pk and author_by_es and str(author_by_pk.id) != str(author_by_es.get("pk_id")): + raise BadRequest("Parameters 'id' and 'pk_id' refer to different users.") + + if author_by_pk and not es_id: + es_id = author_by_pk.json.get("id") + if author_by_es and not pk_id: + pk_id = author_by_es.get("pk_id") + + self.validate_author_data(author_data, pk_id, is_update=True) + + # scheme -> id + author_data = self.process_authors_data_before(author_data) + + # author_data['id'] = es_id + author_data['pk_id'] = pk_id + author_data['gather_flg'] = 0 + + WekoAuthors.update(pk_id,author_data,data.get("force_change",False)) + + doc = current_search_client.get(index=search_index, id=es_id, doc_type="_doc") + result = doc.get("_source", {}) + + # id -> scheme + author_data = self.process_authors_data_after(result) + + return Response( + response=json.dumps({ + "message": "Author successfully updated.", + "author": self.remove_filed_no_need(author_data) + }), + status=200, + content_type='application/json' + ) + + except BadRequest as e: + raise BadRequest(str(e)) + except NotFound as e: + raise NotFound(str(e)) + except SQLAlchemyError: + db.session.rollback() + current_app.logger.error(traceback.format_exc()) + raise InternalServerError("Database error.") + except Exception: + current_app.logger.error(traceback.format_exc()) + raise InternalServerError("Internal server error.") + + def extract_data(self, data): + result = { + "authorIdInfo_idType": [item["idType"] for item in data.get("authorIdInfo", []) if "idType" in item], + "authorNameInfo_language": [item["language"] for item in data.get("authorNameInfo", []) if "language" in item], + "affiliationInfo_affiliationIdType": [ + identifier["affiliationIdType"] + for aff in data.get("affiliationInfo", []) + for identifier in aff.get("identifierInfo", []) + if "affiliationIdType" in identifier + ], + "affiliationInfo_affiliationNameLang": [ + name_info["affiliationNameLang"] + for aff in data.get("affiliationInfo", []) + for name_info in aff.get("affiliationNameInfo", []) + if "affiliationNameLang" in name_info + ] + } + print(f"extract_data:{result}") + return result + + def get_all_schemes(self): + from .models import AuthorsPrefixSettings, AuthorsAffiliationSettings + prefix_schemes = db.session.query(AuthorsPrefixSettings.scheme).distinct().all() + affiliation_schemes = db.session.query(AuthorsAffiliationSettings.scheme).distinct().all() + prefix_schemes = [scheme[0] for scheme in prefix_schemes] + affiliation_schemes = [scheme[0] for scheme in affiliation_schemes] + print(f"prefix_schemes:{prefix_schemes}") + print(f"affiliation_schemes:{affiliation_schemes}") + return prefix_schemes,affiliation_schemes + + + def validate_request_data(self, extracted_data, lang_options_list, prefix_schemes, affiliation_schemes): + if not all(lang in lang_options_list for lang in extracted_data["authorNameInfo_language"]): + print("authorNameInfo_language") + return False + if not all(lang in lang_options_list for lang in extracted_data["affiliationInfo_affiliationNameLang"]): + print("affiliationInfo_affiliationNameLang") + return False + if not all(scheme in prefix_schemes for scheme in extracted_data["authorIdInfo_idType"]): + print("authorIdInfo_idType") + return False + if not all(scheme in affiliation_schemes for scheme in extracted_data["affiliationInfo_affiliationIdType"]): + print("affiliationInfo_affiliationIdType") + return False + return True + + + def delete_v1(self, **kwargs): + """Handle DELETE request for author deletion.""" + from weko_authors.models import Authors + try: + data = request.get_json() + + if not data: + raise BadRequest("Request body cannot be empty.") + + es_id = data.get("id") + pk_id = data.get("pk_id") + + if not es_id and not pk_id: + raise BadRequest("Either 'id' or 'pk_id' must be specified.") + + author_by_pk = None + author_by_es = None + search_index = current_app.config.get("WEKO_AUTHORS_ES_INDEX_NAME") + + if pk_id: + author_by_pk = Authors.query.filter_by(id=pk_id).first() + + if es_id: + search_results = current_search_client.search( + index=search_index, + body={"query": {"term": {"_id": es_id}}}, + size=1 + ) + if search_results["hits"]["total"]> 0: + author_by_es = search_results["hits"]["hits"][0]["_source"] + + if not author_by_pk and not author_by_es: + raise NotFound("Specified author does not exist.") + + if author_by_pk and not es_id: + es_id = author_by_pk.json.get("id") + search_results = current_search_client.search( + index=search_index, + body={"query": {"term": {"_id": es_id}}}, + size=1 + ) + if search_results["hits"]["total"]> 0: + author_by_es = search_results["hits"]["hits"][0]["_source"] + + if author_by_es and not pk_id: + pk_id = author_by_es.get("pk_id") + author_by_pk = Authors.query.filter_by(id=pk_id).first() + + if author_by_pk and author_by_es and str(author_by_pk.id) != str(author_by_es.get("pk_id")): + raise BadRequest("Parameters 'id' and 'pk_id' refer to different users.") + + if author_by_pk: + author_by_pk.is_deleted = True + db.session.commit() + + if author_by_es: + current_search_client.update( + index=search_index, + id=es_id, + body={"doc": {"is_deleted": True}}, + doc_type="_doc" + ) + return self.make_response(200) + + except BadRequest as e: + raise BadRequest(str(e)) + except NotFound as e: + raise NotFound(str(e)) + except SQLAlchemyError: + db.session.rollback() + raise InternalServerError("Database error.") + except Exception as e: + current_app.logger.error(traceback.format_exc()) + raise InternalServerError("Internal server error.") diff --git a/modules/weko-authors/weko_authors/scopes.py b/modules/weko-authors/weko_authors/scopes.py new file mode 100644 index 0000000000..d61d61c283 --- /dev/null +++ b/modules/weko-authors/weko_authors/scopes.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2017-2018 CERN. +# +# Invenio is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""OAuth2 scopes.""" + +from __future__ import absolute_import, print_function + +from flask_babelex import lazy_gettext as _ + +from invenio_oauth2server.models import Scope + +author_search_scope = Scope( + id_='author:search', + group='author', + help_text=_('Allow search'), +) + +author_create_scope = Scope( + id_='author:create', + group='author', + help_text=_('Allow create'), +) + +author_update_scope = Scope( + id_='author:update', + group='author', + help_text=_('Allow update'), +) + +author_delete_scope = Scope( + id_='author:delete', + group='author', + help_text=_('Allow delete'), +) \ No newline at end of file diff --git a/modules/weko-authors/weko_authors/static/css/weko_authors/app-author-export.main.chunk.css b/modules/weko-authors/weko_authors/static/css/weko_authors/app-author-export.main.chunk.css index 94077beb7d..19d7ad1d11 100644 --- a/modules/weko-authors/weko_authors/static/css/weko_authors/app-author-export.main.chunk.css +++ b/modules/weko-authors/weko_authors/static/css/weko_authors/app-author-export.main.chunk.css @@ -1,2 +1,2 @@ -.opacity1{opacity:1}.modal-dialog{margin-top:130px!important}.btn.margin{margin-top:10px;margin-left:10px;min-width:100px;height:35px}.loading{position:relative;display:inline-block;top:3px;right:6px;height:16px;width:16px;border:1px solid rgba(0,0,0,.281);border-top-color:#aee9ff;border-radius:100%;animation:rotation 1.5s linear .25s infinite;opacity:1}@keyframes rotation{0%{opacity:1;transform:rotate(0deg)}to{opacity:1;transform:rotate(359deg)}}.in-line{display:inline} -/*# sourceMappingURL=main.8933ffa8.chunk.css.map */ \ No newline at end of file +.opacity1{opacity:1}.modal-dialog{margin-top:130px!important}.btn.margin{margin-top:10px;margin-left:10px;min-width:100px;height:35px}.select-margin{margin-left:15%;width:10%}.loading{position:relative;display:inline-block;top:3px;right:6px;height:16px;width:16px;border:1px solid rgba(0,0,0,.281);border-top-color:#aee9ff;border-radius:100%;animation:rotation 1.5s linear .25s infinite;opacity:1}@keyframes rotation{0%{opacity:1;transform:rotate(0deg)}to{opacity:1;transform:rotate(359deg)}}.in-line{display:inline} +/*# sourceMappingURL=main.1080a3ea.chunk.css.map */ \ No newline at end of file diff --git a/modules/weko-authors/weko_authors/static/css/weko_authors/app-author-import.main.chunk.css b/modules/weko-authors/weko_authors/static/css/weko_authors/app-author-import.main.chunk.css index 859ee0e6f4..fd98b2869b 100644 --- a/modules/weko-authors/weko_authors/static/css/weko_authors/app-author-import.main.chunk.css +++ b/modules/weko-authors/weko_authors/static/css/weko_authors/app-author-import.main.chunk.css @@ -1,2 +1,2 @@ -.loading{position:relative;display:inline-block;top:3px;right:6px;height:16px;width:16px;border:1px solid rgba(0,0,0,.281);border-top-color:#aee9ff;border-radius:100%;-webkit-animation:rotation 1.5s linear .25s infinite;animation:rotation 1.5s linear .25s infinite;opacity:1}@-webkit-keyframes rotation{0%{opacity:1;-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{opacity:1;-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes rotation{0%{opacity:1;-webkit-transform:rotate(0deg);transform:rotate(0deg)}to{opacity:1;-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.opacity1{opacity:1}.m-t-20{margin-top:20px}.hide{display:none}.tab-content{padding:20px}.icon{padding:0 5px}.block-placeholder{padding:10px 0}.block-placeholder p{color:grey}.block-placeholder p.active{color:#000!important}.cursor-pointer{cursor:pointer}.block-summary{display:flex;align-items:flex-end}.text-align-right{text-align:right}.flex-box{display:flex;justify-content:space-between}.newline{display:block}.table-title-170{min-width:170px;margin:0}.table-title-100{min-width:100px;margin:0}.t_head{margin:0}.end_date,.start_date{width:200px} -/*# sourceMappingURL=main.a2a8b022.chunk.css.map */ \ No newline at end of file +.loading{position:relative;display:inline-block;top:3px;right:6px;height:16px;width:16px;border:1px solid rgba(0,0,0,.281);border-top-color:#aee9ff;border-radius:100%;animation:rotation 1.5s linear .25s infinite;opacity:1}@keyframes rotation{0%{opacity:1;transform:rotate(0deg)}to{opacity:1;transform:rotate(359deg)}}.modal-dialog{margin-top:130px!important}.modal-text{padding-left:20px}.margin{margin-left:15%}.width-10percent{width:10%}.opacity1{opacity:1}.m-t-20{margin-top:20px}.hide{display:none}.tab-content{padding:20px}.icon{padding:0 5px}.block-placeholder{padding:10px 0}.block-placeholder p{color:grey}.block-placeholder p.active{color:#000!important}.cursor-pointer{cursor:pointer}.block-summary{display:flex;align-items:flex-end}.text-align-right{text-align:right}.alignCenter{text-align:center}.flex-box{display:flex;justify-content:space-between}.newline{display:block}.table-title-170{min-width:170px;margin:0}.table-title-100{min-width:100px;margin:0}.t_head{margin:0}.end_date,.start_date{width:200px} +/*# sourceMappingURL=main.2391e4eb.chunk.css.map */ \ No newline at end of file diff --git a/modules/weko-authors/weko_authors/static/js/weko_authors/app-author-export.chunk.js b/modules/weko-authors/weko_authors/static/js/weko_authors/app-author-export.chunk.js index be46847845..f25905b056 100644 --- a/modules/weko-authors/weko_authors/static/js/weko_authors/app-author-export.chunk.js +++ b/modules/weko-authors/weko_authors/static/js/weko_authors/app-author-export.chunk.js @@ -1,3 +1,3 @@ -/*! For license information please see 2.cccf5892.chunk.js.LICENSE.txt */ -(this["webpackJsonpapp-author-export"]=this["webpackJsonpapp-author-export"]||[]).push([[2],[function(e,t,n){"use strict";e.exports=n(391)},function(e,t,n){var r=n(3),o=n(18).f,i=n(23),a=n(20),u=n(104),l=n(134),c=n(67);e.exports=function(e,t){var n,s,f,d,p,h=e.target,v=e.global,y=e.stat;if(n=v?r:y?r[h]||u(h,{}):(r[h]||{}).prototype)for(s in t){if(d=t[s],f=e.noTargetGet?(p=o(n,s))&&p.value:n[s],!c(v?s:h+(y?".":"#")+s,e.forced)&&void 0!==f){if(typeof d===typeof f)continue;l(d,f)}(e.sham||f&&f.sham)&&i(d,"sham",!0),a(n,s,d,e)}}},function(e,t){e.exports=function(e){try{return!!e()}catch(t){return!0}}},function(e,t,n){(function(t){var n=function(e){return e&&e.Math==Math&&e};e.exports=n("object"==typeof globalThis&&globalThis)||n("object"==typeof window&&window)||n("object"==typeof self&&self)||n("object"==typeof t&&t)||function(){return this}()||Function("return this")()}).call(this,n(131))},function(e,t,n){"use strict";e.exports=n(397)},function(e,t,n){"use strict";function r(){return(r=Object.assign||function(e){for(var t=1;t0?o(r(e),9007199254740991):0}},function(e,t,n){var r=n(11),o=n(132),i=n(7),a=n(35),u=Object.defineProperty;t.f=r?u:function(e,t,n){if(i(e),t=a(t,!0),i(n),o)try{return u(e,t,n)}catch(r){}if("get"in n||"set"in n)throw TypeError("Accessors not supported");return"value"in n&&(e[t]=n.value),e}},function(e,t,n){var r=n(22);e.exports=function(e){return Object(r(e))}},function(e,t,n){"use strict";function r(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r=0||(o[n]=e[n]);return o}n.d(t,"a",(function(){return r}))},function(e,t){var n={}.hasOwnProperty;e.exports=function(e,t){return n.call(e,t)}},function(e,t,n){var r;!function(){"use strict";var n={}.hasOwnProperty;function o(){for(var e=[],t=0;tk;k++)if((p||k in x)&&(b=E(m=x[k],k,w),e))if(t)_[k]=b;else if(b)switch(e){case 3:return!0;case 5:return m;case 6:return k;case 2:l.call(_,m)}else switch(e){case 4:return!1;case 7:l.call(_,m)}return f?-1:c||s?s:_}};e.exports={forEach:c(0),map:c(1),filter:c(2),some:c(3),every:c(4),find:c(5),findIndex:c(6),filterOut:c(7)}},function(e,t){e.exports=function(e){if(void 0==e)throw TypeError("Can't call method on "+e);return e}},function(e,t,n){var r=n(11),o=n(13),i=n(42);e.exports=r?function(e,t,n){return o.f(e,t,i(1,n))}:function(e,t,n){return e[t]=n,e}},function(e,t,n){var r=n(63),o=n(22);e.exports=function(e){return r(o(e))}},function(e,t,n){var r,o,i,a=n(133),u=n(3),l=n(6),c=n(23),s=n(16),f=n(106),d=n(81),p=n(65),h=u.WeakMap;if(a){var v=f.state||(f.state=new h),y=v.get,g=v.has,m=v.set;r=function(e,t){return t.facade=e,m.call(v,e,t),t},o=function(e){return y.call(v,e)||{}},i=function(e){return g.call(v,e)}}else{var b=d("state");p[b]=!0,r=function(e,t){return t.facade=e,c(e,b,t),t},o=function(e){return s(e,b)?e[b]:{}},i=function(e){return s(e,b)}}e.exports={set:r,get:o,has:i,enforce:function(e){return i(e)?o(e):r(e,{})},getterFor:function(e){return function(t){var n;if(!l(t)||(n=o(t)).type!==e)throw TypeError("Incompatible receiver, "+e+" required");return n}}}},function(e,t,n){var r=n(22),o=/"/g;e.exports=function(e,t,n,i){var a=String(r(e)),u="<"+t;return""!==n&&(u+=" "+n+'="'+String(i).replace(o,""")+'"'),u+">"+a+""}},function(e,t,n){var r=n(2);e.exports=function(e){return r((function(){var t=""[e]('"');return t!==t.toLowerCase()||t.split('"').length>3}))}},function(e,t,n){"use strict";n.d(t,"a",(function(){return i}));var r=n(0);var o=function(e){var t=Object(r.useRef)(e);return Object(r.useEffect)((function(){t.current=e}),[e]),t};function i(e){var t=o(e);return Object(r.useCallback)((function(){return t.current&&t.current.apply(t,arguments)}),[t])}},function(e,t){var n=Math.ceil,r=Math.floor;e.exports=function(e){return isNaN(e=+e)?0:(e>0?r:n)(e)}},function(e,t){e.exports=function(e){if("function"!=typeof e)throw TypeError(String(e)+" is not a function");return e}},function(e,t,n){var r=n(135),o=n(3),i=function(e){return"function"==typeof e?e:void 0};e.exports=function(e,t){return arguments.length<2?i(r[e])||i(o[e]):r[e]&&r[e][t]||o[e]&&o[e][t]}},function(e,t,n){var r=n(16),o=n(14),i=n(81),a=n(116),u=i("IE_PROTO"),l=Object.prototype;e.exports=a?Object.getPrototypeOf:function(e){return e=o(e),r(e,u)?e[u]:"function"==typeof e.constructor&&e instanceof e.constructor?e.constructor.prototype:e instanceof Object?l:null}},function(e,t,n){"use strict";function r(e){return e&&e.ownerDocument||document}n.d(t,"a",(function(){return r}))},function(e,t,n){"use strict";var r=n(33);function o(e,t){return function(e){var t=Object(r.a)(e);return t&&t.defaultView||window}(e).getComputedStyle(e,t)}var i=/([A-Z])/g;var a=/^ms-/;function u(e){return function(e){return e.replace(i,"-$1").toLowerCase()}(e).replace(a,"-ms-")}var l=/^((translate|rotate|scale)(X|Y|Z|3d)?|matrix(3d)?|perspective|skew(X|Y)?)$/i;t.a=function(e,t){var n="",r="";if("string"===typeof t)return e.style.getPropertyValue(u(t))||o(e).getPropertyValue(u(t));Object.keys(t).forEach((function(o){var i=t[o];i||0===i?!function(e){return!(!e||!l.test(e))}(o)?n+=u(o)+": "+i+";":r+=o+"("+i+") ":e.style.removeProperty(u(o))})),r&&(n+="transform: "+r+";"),e.style.cssText+=";"+n}},function(e,t,n){var r=n(6);e.exports=function(e,t){if(!r(e))return e;var n,o;if(t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;if("function"==typeof(n=e.valueOf)&&!r(o=n.call(e)))return o;if(!t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;throw TypeError("Can't convert object to primitive value")}},function(e,t){e.exports=!1},function(e,t,n){var r=n(13).f,o=n(16),i=n(9)("toStringTag");e.exports=function(e,t,n){e&&!o(e=n?e:e.prototype,i)&&r(e,i,{configurable:!0,value:t})}},function(e,t,n){"use strict";n.d(t,"a",(function(){return a}));n(5);var r=n(0),o=n.n(r),i=o.a.createContext({});i.Consumer,i.Provider;function a(e,t){var n=Object(r.useContext)(i);return e||n[t]||t}},function(e,t,n){"use strict";var r=n(1),o=n(3),i=n(11),a=n(127),u=n(10),l=n(169),c=n(47),s=n(42),f=n(23),d=n(12),p=n(171),h=n(186),v=n(35),y=n(16),g=n(75),m=n(6),b=n(44),w=n(56),x=n(50).f,E=n(187),S=n(21).forEach,k=n(58),O=n(13),_=n(18),T=n(25),C=n(89),P=T.get,j=T.set,N=O.f,A=_.f,R=Math.round,L=o.RangeError,I=l.ArrayBuffer,M=l.DataView,F=u.NATIVE_ARRAY_BUFFER_VIEWS,U=u.TYPED_ARRAY_TAG,D=u.TypedArray,z=u.TypedArrayPrototype,B=u.aTypedArrayConstructor,V=u.isTypedArray,W="BYTES_PER_ELEMENT",H="Wrong length",$=function(e,t){for(var n=0,r=t.length,o=new(B(e))(r);r>n;)o[n]=t[n++];return o},q=function(e,t){N(e,t,{get:function(){return P(this)[t]}})},Q=function(e){var t;return e instanceof I||"ArrayBuffer"==(t=g(e))||"SharedArrayBuffer"==t},Y=function(e,t){return V(e)&&"symbol"!=typeof t&&t in e&&String(+t)==String(t)},G=function(e,t){return Y(e,t=v(t,!0))?s(2,e[t]):A(e,t)},K=function(e,t,n){return!(Y(e,t=v(t,!0))&&m(n)&&y(n,"value"))||y(n,"get")||y(n,"set")||n.configurable||y(n,"writable")&&!n.writable||y(n,"enumerable")&&!n.enumerable?N(e,t,n):(e[t]=n.value,e)};i?(F||(_.f=G,O.f=K,q(z,"buffer"),q(z,"byteOffset"),q(z,"byteLength"),q(z,"length")),r({target:"Object",stat:!0,forced:!F},{getOwnPropertyDescriptor:G,defineProperty:K}),e.exports=function(e,t,n){var i=e.match(/\d+$/)[0]/8,u=e+(n?"Clamped":"")+"Array",l="get"+e,s="set"+e,v=o[u],y=v,g=y&&y.prototype,O={},_=function(e,t){N(e,t,{get:function(){return function(e,t){var n=P(e);return n.view[l](t*i+n.byteOffset,!0)}(this,t)},set:function(e){return function(e,t,r){var o=P(e);n&&(r=(r=R(r))<0?0:r>255?255:255&r),o.view[s](t*i+o.byteOffset,r,!0)}(this,t,e)},enumerable:!0})};F?a&&(y=t((function(e,t,n,r){return c(e,y,u),C(m(t)?Q(t)?void 0!==r?new v(t,h(n,i),r):void 0!==n?new v(t,h(n,i)):new v(t):V(t)?$(y,t):E.call(y,t):new v(p(t)),e,y)})),w&&w(y,D),S(x(v),(function(e){e in y||f(y,e,v[e])})),y.prototype=g):(y=t((function(e,t,n,r){c(e,y,u);var o,a,l,s=0,f=0;if(m(t)){if(!Q(t))return V(t)?$(y,t):E.call(y,t);o=t,f=h(n,i);var v=t.byteLength;if(void 0===r){if(v%i)throw L(H);if((a=v-f)<0)throw L(H)}else if((a=d(r)*i)+f>v)throw L(H);l=a/i}else l=p(t),o=new I(a=l*i);for(j(e,{buffer:o,byteOffset:f,byteLength:a,length:l,view:new M(o)});s"+e+""},h=function(){try{r=document.domain&&new ActiveXObject("htmlfile")}catch(t){}h=r?function(e){e.write(p("")),e.close();var t=e.parentWindow.Object;return e=null,t}(r):function(){var e,t=c("iframe");return t.style.display="none",l.appendChild(t),t.src=String("javascript:"),(e=t.contentWindow.document).open(),e.write(p("document.F=Object")),e.close(),e.F}();for(var e=a.length;e--;)delete h.prototype[a[e]];return h()};u[f]=!0,e.exports=Object.create||function(e,t){var n;return null!==e?(d.prototype=o(e),n=new d,d.prototype=null,n[f]=e):n=h(),void 0===t?n:i(n,t)}},function(e,t,n){var r=n(30);e.exports=function(e,t,n){if(r(e),void 0===t)return e;switch(n){case 0:return function(){return e.call(t)};case 1:return function(n){return e.call(t,n)};case 2:return function(n,r){return e.call(t,n,r)};case 3:return function(n,r,o){return e.call(t,n,r,o)}}return function(){return e.apply(t,arguments)}}},function(e,t,n){var r=n(9),o=n(44),i=n(13),a=r("unscopables"),u=Array.prototype;void 0==u[a]&&i.f(u,a,{configurable:!0,value:o(null)}),e.exports=function(e){u[a][e]=!0}},function(e,t){e.exports=function(e,t,n){if(!(e instanceof t))throw TypeError("Incorrect "+(n?n+" ":"")+"invocation");return e}},function(e,t,n){"use strict";t.a=!("undefined"===typeof window||!window.document||!window.document.createElement)},function(e,t,n){"use strict";!function e(){if("undefined"!==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__&&"function"===typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE)try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(e)}catch(t){console.error(t)}}(),e.exports=n(392)},function(e,t,n){var r=n(136),o=n(109).concat("length","prototype");t.f=Object.getOwnPropertyNames||function(e){return r(e,o)}},function(e,t,n){var r=n(29),o=Math.max,i=Math.min;e.exports=function(e,t){var n=r(e);return n<0?o(n+t,0):i(n,t)}},function(e,t,n){"use strict";var r=n(35),o=n(13),i=n(42);e.exports=function(e,t,n){var a=r(t);a in e?o.f(e,a,i(0,n)):e[a]=n}},function(e,t,n){"use strict";var r=n(2);e.exports=function(e,t){var n=[][e];return!!n&&r((function(){n.call(null,t||function(){throw 1},1)}))}},function(e,t,n){var r=n(65),o=n(6),i=n(16),a=n(13).f,u=n(64),l=n(76),c=u("meta"),s=0,f=Object.isExtensible||function(){return!0},d=function(e){a(e,c,{value:{objectID:"O"+ ++s,weakData:{}}})},p=e.exports={REQUIRED:!1,fastKey:function(e,t){if(!o(e))return"symbol"==typeof e?e:("string"==typeof e?"S":"P")+e;if(!i(e,c)){if(!f(e))return"F";if(!t)return"E";d(e)}return e[c].objectID},getWeakData:function(e,t){if(!i(e,c)){if(!f(e))return!0;if(!t)return!1;d(e)}return e[c].weakData},onFreeze:function(e){return l&&p.REQUIRED&&f(e)&&!i(e,c)&&d(e),e}};r[c]=!0},function(e,t,n){var r=n(7),o=n(30),i=n(9)("species");e.exports=function(e,t){var n,a=r(e).constructor;return void 0===a||void 0==(n=r(a)[i])?t:o(n)}},function(e,t,n){var r=n(7),o=n(164);e.exports=Object.setPrototypeOf||("__proto__"in{}?function(){var e,t=!1,n={};try{(e=Object.getOwnPropertyDescriptor(Object.prototype,"__proto__").set).call(n,[]),t=n instanceof Array}catch(i){}return function(n,i){return r(n),o(i),t?e.call(n,i):n.__proto__=i,n}}():void 0)},function(e,t,n){var r=n(43),o=n(3);e.exports="process"==r(o.process)},function(e,t,n){"use strict";var r=n(31),o=n(13),i=n(9),a=n(11),u=i("species");e.exports=function(e){var t=r(e),n=o.f;a&&t&&!t[u]&&n(t,u,{configurable:!0,get:function(){return this}})}},function(e,t,n){var r=n(20);e.exports=function(e,t,n){for(var o in t)r(e,o,t[o],n);return e}},function(e,t,n){var r=n(22),o="["+n(91)+"]",i=RegExp("^"+o+o+"*"),a=RegExp(o+o+"*$"),u=function(e){return function(t){var n=String(r(t));return 1&e&&(n=n.replace(i,"")),2&e&&(n=n.replace(a,"")),n}};e.exports={start:u(1),end:u(2),trim:u(3)}},function(e,t,n){"use strict";var r=n(97),o=n(78);t.a=function(e,t,n,i){return Object(r.a)(e,t,n,i),function(){Object(o.a)(e,t,n,i)}}},function(e,t,n){"use strict";var r=n(5),o=n(15),i=n(17),a=n.n(i),u=n(0),l=n.n(u);function c(e,t){return(c=Object.setPrototypeOf||function(e,t){return e.__proto__=t,e})(e,t)}n(8);var s=n(49),f=n.n(s),d=!1,p=l.a.createContext(null),h="unmounted",v="exited",y="entering",g="entered",m="exiting",b=function(e){var t,n;function r(t,n){var r;r=e.call(this,t,n)||this;var o,i=n&&!n.isMounting?t.enter:t.appear;return r.appearStatus=null,t.in?i?(o=v,r.appearStatus=y):o=g:o=t.unmountOnExit||t.mountOnEnter?h:v,r.state={status:o},r.nextCallback=null,r}n=e,(t=r).prototype=Object.create(n.prototype),t.prototype.constructor=t,c(t,n),r.getDerivedStateFromProps=function(e,t){return e.in&&t.status===h?{status:v}:null};var o=r.prototype;return o.componentDidMount=function(){this.updateStatus(!0,this.appearStatus)},o.componentDidUpdate=function(e){var t=null;if(e!==this.props){var n=this.state.status;this.props.in?n!==y&&n!==g&&(t=y):n!==y&&n!==g||(t=m)}this.updateStatus(!1,t)},o.componentWillUnmount=function(){this.cancelNextCallback()},o.getTimeouts=function(){var e,t,n,r=this.props.timeout;return e=t=n=r,null!=r&&"number"!==typeof r&&(e=r.exit,t=r.enter,n=void 0!==r.appear?r.appear:t),{exit:e,enter:t,appear:n}},o.updateStatus=function(e,t){void 0===e&&(e=!1),null!==t?(this.cancelNextCallback(),t===y?this.performEnter(e):this.performExit()):this.props.unmountOnExit&&this.state.status===v&&this.setState({status:h})},o.performEnter=function(e){var t=this,n=this.props.enter,r=this.context?this.context.isMounting:e,o=this.props.nodeRef?[r]:[f.a.findDOMNode(this),r],i=o[0],a=o[1],u=this.getTimeouts(),l=r?u.appear:u.enter;!e&&!n||d?this.safeSetState({status:g},(function(){t.props.onEntered(i)})):(this.props.onEnter(i,a),this.safeSetState({status:y},(function(){t.props.onEntering(i,a),t.onTransitionEnd(l,(function(){t.safeSetState({status:g},(function(){t.props.onEntered(i,a)}))}))})))},o.performExit=function(){var e=this,t=this.props.exit,n=this.getTimeouts(),r=this.props.nodeRef?void 0:f.a.findDOMNode(this);t&&!d?(this.props.onExit(r),this.safeSetState({status:m},(function(){e.props.onExiting(r),e.onTransitionEnd(n.exit,(function(){e.safeSetState({status:v},(function(){e.props.onExited(r)}))}))}))):this.safeSetState({status:v},(function(){e.props.onExited(r)}))},o.cancelNextCallback=function(){null!==this.nextCallback&&(this.nextCallback.cancel(),this.nextCallback=null)},o.safeSetState=function(e,t){t=this.setNextCallback(t),this.setState(e,t)},o.setNextCallback=function(e){var t=this,n=!0;return this.nextCallback=function(r){n&&(n=!1,t.nextCallback=null,e(r))},this.nextCallback.cancel=function(){n=!1},this.nextCallback},o.onTransitionEnd=function(e,t){this.setNextCallback(t);var n=this.props.nodeRef?this.props.nodeRef.current:f.a.findDOMNode(this),r=null==e&&!this.props.addEndListener;if(n&&!r){if(this.props.addEndListener){var o=this.props.nodeRef?[this.nextCallback]:[n,this.nextCallback],i=o[0],a=o[1];this.props.addEndListener(i,a)}null!=e&&setTimeout(this.nextCallback,e)}else setTimeout(this.nextCallback,0)},o.render=function(){var e=this.state.status;if(e===h)return null;var t=this.props,n=t.children,r=(t.in,t.mountOnEnter,t.unmountOnExit,t.appear,t.enter,t.exit,t.timeout,t.addEndListener,t.onEnter,t.onEntering,t.onEntered,t.onExit,t.onExiting,t.onExited,t.nodeRef,function(e,t){if(null==e)return{};var n,r,o={},i=Object.keys(e);for(r=0;r=0||(o[n]=e[n]);return o}(t,["children","in","mountOnEnter","unmountOnExit","appear","enter","exit","timeout","addEndListener","onEnter","onEntering","onEntered","onExit","onExiting","onExited","nodeRef"]));return l.a.createElement(p.Provider,{value:null},"function"===typeof n?n(e,r):l.a.cloneElement(l.a.Children.only(n),r))},r}(l.a.Component);function w(){}b.contextType=p,b.propTypes={},b.defaultProps={in:!1,mountOnEnter:!1,unmountOnExit:!1,appear:!1,enter:!0,exit:!0,onEnter:w,onEntering:w,onEntered:w,onExit:w,onExiting:w,onExited:w},b.UNMOUNTED=h,b.EXITED=v,b.ENTERING=y,b.ENTERED=g,b.EXITING=m;var x=b,E=n(34),S=n(101);function k(e,t){var n=Object(E.a)(e,t)||"",r=-1===n.indexOf("ms")?1e3:1;return parseFloat(n)*r}function O(e,t){var n=k(e,"transitionDuration"),r=k(e,"transitionDelay"),o=Object(S.a)(e,(function(n){n.target===e&&(o(),t(n))}),n+r)}var _,T=["className","children"],C=((_={}).entering="show",_.entered="show",_),P=l.a.forwardRef((function(e,t){var n=e.className,i=e.children,c=Object(o.a)(e,T),s=Object(u.useCallback)((function(e){!function(e){e.offsetHeight}(e),c.onEnter&&c.onEnter(e)}),[c]);return l.a.createElement(x,Object(r.a)({ref:t,addEndListener:O},c,{onEnter:s}),(function(e,t){return l.a.cloneElement(i,Object(r.a)({},t,{className:a()("fade",n,i.props.className,C[e])}))}))}));P.defaultProps={in:!1,timeout:300,mountOnEnter:!1,unmountOnExit:!1,appear:!1},P.displayName="Fade";t.a=P},function(e,t,n){var r=n(2),o=n(43),i="".split;e.exports=r((function(){return!Object("z").propertyIsEnumerable(0)}))?function(e){return"String"==o(e)?i.call(e,""):Object(e)}:Object},function(e,t){var n=0,r=Math.random();e.exports=function(e){return"Symbol("+String(void 0===e?"":e)+")_"+(++n+r).toString(36)}},function(e,t){e.exports={}},function(e,t,n){var r=n(24),o=n(12),i=n(51),a=function(e){return function(t,n,a){var u,l=r(t),c=o(l.length),s=i(a,c);if(e&&n!=n){for(;c>s;)if((u=l[s++])!=u)return!0}else for(;c>s;s++)if((e||s in l)&&l[s]===n)return e||s||0;return!e&&-1}};e.exports={includes:a(!0),indexOf:a(!1)}},function(e,t,n){var r=n(2),o=/#|\.prototype\./,i=function(e,t){var n=u[a(e)];return n==c||n!=l&&("function"==typeof t?r(t):!!t)},a=i.normalize=function(e){return String(e).replace(o,".").toLowerCase()},u=i.data={},l=i.NATIVE="N",c=i.POLYFILL="P";e.exports=i},function(e,t,n){var r=n(43);e.exports=Array.isArray||function(e){return"Array"==r(e)}},function(e,t,n){var r=n(136),o=n(109);e.exports=Object.keys||function(e){return r(e,o)}},function(e,t,n){var r=n(6),o=n(68),i=n(9)("species");e.exports=function(e,t){var n;return o(e)&&("function"!=typeof(n=e.constructor)||n!==Array&&!o(n.prototype)?r(n)&&null===(n=n[i])&&(n=void 0):n=void 0),new(void 0===n?Array:n)(0===t?0:t)}},function(e,t,n){var r=n(2),o=n(9),i=n(72),a=o("species");e.exports=function(e){return i>=51||!r((function(){var t=[];return(t.constructor={})[a]=function(){return{foo:1}},1!==t[e](Boolean).foo}))}},function(e,t,n){var r,o,i=n(3),a=n(82),u=i.process,l=u&&u.versions,c=l&&l.v8;c?o=(r=c.split("."))[0]+r[1]:a&&(!(r=a.match(/Edge\/(\d+)/))||r[1]>=74)&&(r=a.match(/Chrome\/(\d+)/))&&(o=r[1]),e.exports=o&&+o},function(e,t){e.exports={}},function(e,t,n){var r=n(75),o=n(73),i=n(9)("iterator");e.exports=function(e){if(void 0!=e)return e[i]||e["@@iterator"]||o[r(e)]}},function(e,t,n){var r=n(113),o=n(43),i=n(9)("toStringTag"),a="Arguments"==o(function(){return arguments}());e.exports=r?o:function(e){var t,n,r;return void 0===e?"Undefined":null===e?"Null":"string"==typeof(n=function(e,t){try{return e[t]}catch(n){}}(t=Object(e),i))?n:a?o(t):"Object"==(r=o(t))&&"function"==typeof t.callee?"Arguments":r}},function(e,t,n){var r=n(2);e.exports=!r((function(){return Object.isExtensible(Object.preventExtensions({}))}))},function(e,t,n){var r=n(7),o=n(112),i=n(12),a=n(45),u=n(74),l=n(158),c=function(e,t){this.stopped=e,this.result=t};e.exports=function(e,t,n){var s,f,d,p,h,v,y,g=n&&n.that,m=!(!n||!n.AS_ENTRIES),b=!(!n||!n.IS_ITERATOR),w=!(!n||!n.INTERRUPTED),x=a(t,g,1+m+w),E=function(e){return s&&l(s),new c(!0,e)},S=function(e){return m?(r(e),w?x(e[0],e[1],E):x(e[0],e[1])):w?x(e,E):x(e)};if(b)s=e;else{if("function"!=typeof(f=u(e)))throw TypeError("Target is not iterable");if(o(f)){for(d=0,p=i(e.length);p>d;d++)if((h=S(e[d]))&&h instanceof c)return h;return new c(!1)}s=f.call(e)}for(v=s.next;!(y=v.call(s)).done;){try{h=S(y.value)}catch(k){throw l(s),k}if("object"==typeof h&&h&&h instanceof c)return h}return new c(!1)}},function(e,t,n){"use strict";t.a=function(e,t,n,r){var o=r&&"boolean"!==typeof r?r.capture:r;e.removeEventListener(t,n,o),n.__once&&e.removeEventListener(t,n.__once,o)}},function(e,t,n){"use strict";function r(e,t,n,r,o,i,a){try{var u=e[i](a),l=u.value}catch(c){return void n(c)}u.done?t(l):Promise.resolve(l).then(r,o)}function o(e){return function(){var t=this,n=arguments;return new Promise((function(o,i){var a=e.apply(t,n);function u(e){r(a,o,i,u,l,"next",e)}function l(e){r(a,o,i,u,l,"throw",e)}u(void 0)}))}}n.d(t,"a",(function(){return o}))},function(e,t,n){"use strict";var r={}.propertyIsEnumerable,o=Object.getOwnPropertyDescriptor,i=o&&!r.call({1:2},1);t.f=i?function(e){var t=o(this,e);return!!t&&t.enumerable}:r},function(e,t,n){var r=n(107),o=n(64),i=r("keys");e.exports=function(e){return i[e]||(i[e]=o(e))}},function(e,t,n){var r=n(31);e.exports=r("navigator","userAgent")||""},function(e,t,n){var r=n(9)("iterator"),o=!1;try{var i=0,a={next:function(){return{done:!!i++}},return:function(){o=!0}};a[r]=function(){return this},Array.from(a,(function(){throw 2}))}catch(u){}e.exports=function(e,t){if(!t&&!o)return!1;var n=!1;try{var i={};i[r]=function(){return{next:function(){return{done:n=!0}}}},e(i)}catch(u){}return n}},function(e,t,n){var r=n(29),o=n(22),i=function(e){return function(t,n){var i,a,u=String(o(t)),l=r(n),c=u.length;return l<0||l>=c?e?"":void 0:(i=u.charCodeAt(l))<55296||i>56319||l+1===c||(a=u.charCodeAt(l+1))<56320||a>57343?e?u.charAt(l):i:e?u.slice(l,l+2):a-56320+(i-55296<<10)+65536}};e.exports={codeAt:i(!1),charAt:i(!0)}},function(e,t,n){"use strict";var r=Object.getOwnPropertySymbols,o=Object.prototype.hasOwnProperty,i=Object.prototype.propertyIsEnumerable;function a(e){if(null===e||void 0===e)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map((function(e){return t[e]})).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach((function(e){r[e]=e})),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},r)).join("")}catch(o){return!1}}()?Object.assign:function(e,t){for(var n,u,l=a(e),c=1;c=t.length?(e.target=void 0,{value:void 0,done:!0}):"keys"==n?{value:r,done:!1}:"values"==n?{value:t[r],done:!1}:{value:[r,t[r]],done:!1}}),"values"),i.Arguments=i.Array,o("keys"),o("values"),o("entries")},function(e,t,n){var r=n(30),o=n(14),i=n(63),a=n(12),u=function(e){return function(t,n,u,l){r(n);var c=o(t),s=i(c),f=a(c.length),d=e?f-1:0,p=e?-1:1;if(u<2)for(;;){if(d in s){l=s[d],d+=p;break}if(d+=p,e?d<0:f<=d)throw TypeError("Reduce of empty array with no initial value")}for(;e?d>=0:f>d;d+=p)d in s&&(l=n(l,s[d],d,c));return l}};e.exports={left:u(!1),right:u(!0)}},function(e,t,n){"use strict";var r=n(1),o=n(3),i=n(67),a=n(20),u=n(54),l=n(77),c=n(47),s=n(6),f=n(2),d=n(83),p=n(37),h=n(89);e.exports=function(e,t,n){var v=-1!==e.indexOf("Map"),y=-1!==e.indexOf("Weak"),g=v?"set":"add",m=o[e],b=m&&m.prototype,w=m,x={},E=function(e){var t=b[e];a(b,e,"add"==e?function(e){return t.call(this,0===e?0:e),this}:"delete"==e?function(e){return!(y&&!s(e))&&t.call(this,0===e?0:e)}:"get"==e?function(e){return y&&!s(e)?void 0:t.call(this,0===e?0:e)}:"has"==e?function(e){return!(y&&!s(e))&&t.call(this,0===e?0:e)}:function(e,n){return t.call(this,0===e?0:e,n),this})};if(i(e,"function"!=typeof m||!(y||b.forEach&&!f((function(){(new m).entries().next()})))))w=n.getConstructor(t,e,v,g),u.REQUIRED=!0;else if(i(e,!0)){var S=new w,k=S[g](y?{}:-0,1)!=S,O=f((function(){S.has(1)})),_=d((function(e){new m(e)})),T=!y&&f((function(){for(var e=new m,t=5;t--;)e[g](t,t);return!e.has(-0)}));_||((w=t((function(t,n){c(t,w,e);var r=h(new m,t,w);return void 0!=n&&l(n,r[g],{that:r,AS_ENTRIES:v}),r}))).prototype=b,b.constructor=w),(O||T)&&(E("delete"),E("has"),v&&E("get")),(T||k)&&E(g),y&&b.clear&&delete b.clear}return x[e]=w,r({global:!0,forced:w!=m},x),p(w,e),y||n.setStrong(w,e,v),w}},function(e,t,n){var r=n(6),o=n(56);e.exports=function(e,t,n){var i,a;return o&&"function"==typeof(i=t.constructor)&&i!==n&&r(a=i.prototype)&&a!==n.prototype&&o(e,a),e}},function(e,t){var n=Math.expm1,r=Math.exp;e.exports=!n||n(10)>22025.465794806718||n(10)<22025.465794806718||-2e-17!=n(-2e-17)?function(e){return 0==(e=+e)?e:e>-1e-6&&e<1e-6?e+e*e/2:r(e)-1}:n},function(e,t){e.exports="\t\n\v\f\r \xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000\u2028\u2029\ufeff"},function(e,t,n){"use strict";var r=n(36),o=n(3),i=n(2);e.exports=r||!i((function(){var e=Math.random();__defineSetter__.call(null,e,(function(){})),delete o[e]}))},function(e,t,n){"use strict";var r=n(7);e.exports=function(){var e=r(this),t="";return e.global&&(t+="g"),e.ignoreCase&&(t+="i"),e.multiline&&(t+="m"),e.dotAll&&(t+="s"),e.unicode&&(t+="u"),e.sticky&&(t+="y"),t}},function(e,t,n){"use strict";var r=n(93),o=n(122),i=RegExp.prototype.exec,a=String.prototype.replace,u=i,l=function(){var e=/a/,t=/b*/g;return i.call(e,"a"),i.call(t,"a"),0!==e.lastIndex||0!==t.lastIndex}(),c=o.UNSUPPORTED_Y||o.BROKEN_CARET,s=void 0!==/()??/.exec("")[1];(l||s||c)&&(u=function(e){var t,n,o,u,f=this,d=c&&f.sticky,p=r.call(f),h=f.source,v=0,y=e;return d&&(-1===(p=p.replace("y","")).indexOf("g")&&(p+="g"),y=String(e).slice(f.lastIndex),f.lastIndex>0&&(!f.multiline||f.multiline&&"\n"!==e[f.lastIndex-1])&&(h="(?: "+h+")",y=" "+y,v++),n=new RegExp("^(?:"+h+")",p)),s&&(n=new RegExp("^"+h+"$(?!\\s)",p)),l&&(t=f.lastIndex),o=i.call(d?n:f,y),d?o?(o.input=o.input.slice(v),o[0]=o[0].slice(v),o.index=f.lastIndex,f.lastIndex+=o[0].length):f.lastIndex=0:l&&o&&(f.lastIndex=f.global?o.index+o[0].length:t),s&&o&&o.length>1&&a.call(o[0],n,(function(){for(u=1;u")})),s="$0"==="a".replace(/./,"$0"),f=i("replace"),d=!!/./[f]&&""===/./[f]("a","$0"),p=!o((function(){var e=/(?:)/,t=e.exec;e.exec=function(){return t.apply(this,arguments)};var n="ab".split(e);return 2!==n.length||"a"!==n[0]||"b"!==n[1]}));e.exports=function(e,t,n,f){var h=i(e),v=!o((function(){var t={};return t[h]=function(){return 7},7!=""[e](t)})),y=v&&!o((function(){var t=!1,n=/a/;return"split"===e&&((n={}).constructor={},n.constructor[l]=function(){return n},n.flags="",n[h]=/./[h]),n.exec=function(){return t=!0,null},n[h](""),!t}));if(!v||!y||"replace"===e&&(!c||!s||d)||"split"===e&&!p){var g=/./[h],m=n(h,""[e],(function(e,t,n,r,o){return t.exec===a?v&&!o?{done:!0,value:g.call(t,n,r)}:{done:!0,value:e.call(n,t,r)}:{done:!1}}),{REPLACE_KEEPS_$0:s,REGEXP_REPLACE_SUBSTITUTES_UNDEFINED_CAPTURE:d}),b=m[0],w=m[1];r(String.prototype,e,b),r(RegExp.prototype,h,2==t?function(e,t){return w.call(e,this,t)}:function(e){return w.call(e,this)})}f&&u(RegExp.prototype[h],"sham",!0)}},function(e,t,n){var r=n(43),o=n(94);e.exports=function(e,t){var n=e.exec;if("function"===typeof n){var i=n.call(e,t);if("object"!==typeof i)throw TypeError("RegExp exec method returned something other than an Object or null");return i}if("RegExp"!==r(e))throw TypeError("RegExp#exec called on incompatible receiver");return o.call(e,t)}},function(e,t,n){"use strict";var r=n(48),o=!1,i=!1;try{var a={get passive(){return o=!0},get once(){return i=o=!0}};r.a&&(window.addEventListener("test",a,a),window.removeEventListener("test",a,!0))}catch(u){}t.a=function(e,t,n,r){if(r&&"boolean"!==typeof r&&!i){var a=r.once,u=r.capture,l=n;!i&&a&&(l=n.__once||function e(r){this.removeEventListener(t,e,u),n.call(this,r)},n.__once=l),e.addEventListener(t,l,o?r:u)}e.addEventListener(t,n,r)}},function(e,t,n){"use strict";var r=n(5),o=n(15),i=n(8),a=n.n(i),u=n(0),l=n.n(u),c=n(17),s=n.n(c),f=["label","onClick","className"],d={label:a.a.string.isRequired,onClick:a.a.func},p=l.a.forwardRef((function(e,t){var n=e.label,i=e.onClick,a=e.className,u=Object(o.a)(e,f);return l.a.createElement("button",Object(r.a)({ref:t,type:"button",className:s()("close",a),onClick:i},u),l.a.createElement("span",{"aria-hidden":"true"},"\xd7"),l.a.createElement("span",{className:"sr-only"},n))}));p.displayName="CloseButton",p.propTypes=d,p.defaultProps={label:"Close"},t.a=p},function(e,t,n){"use strict";var r=n(5),o=n(0),i=n.n(o),a=n(17),u=n.n(a);t.a=function(e){return i.a.forwardRef((function(t,n){return i.a.createElement("div",Object(r.a)({},t,{ref:n,className:u()(t.className,e)}))}))}},function(e,t,n){"use strict";function r(e,t){return(r=Object.setPrototypeOf||function(e,t){return e.__proto__=t,e})(e,t)}function o(e,t){e.prototype=Object.create(t.prototype),e.prototype.constructor=e,r(e,t)}n.d(t,"a",(function(){return o}))},function(e,t,n){"use strict";n.d(t,"a",(function(){return a}));var r=n(34),o=n(61);function i(e,t,n){void 0===n&&(n=5);var r=!1,i=setTimeout((function(){r||function(e,t,n,r){if(void 0===n&&(n=!1),void 0===r&&(r=!0),e){var o=document.createEvent("HTMLEvents");o.initEvent(t,n,r),e.dispatchEvent(o)}}(e,"transitionend",!0)}),t+n),a=Object(o.a)(e,"transitionend",(function(){r=!0}),{once:!0});return function(){clearTimeout(i),a()}}function a(e,t,n,a){null==n&&(n=function(e){var t=Object(r.a)(e,"transitionDuration")||"",n=-1===t.indexOf("ms")?1e3:1;return parseFloat(t)*n}(e)||0);var u=i(e,n,a),l=Object(o.a)(e,"transitionend",t);return function(){u(),l()}}},function(e,t,n){"use strict";n.d(t,"a",(function(){return o}));var r=n(0);function o(e){var t=function(e){var t=Object(r.useRef)(e);return t.current=e,t}(e);Object(r.useEffect)((function(){return function(){return t.current()}}),[])}},function(e,t,n){var r=n(3),o=n(6),i=r.document,a=o(i)&&o(i.createElement);e.exports=function(e){return a?i.createElement(e):{}}},function(e,t,n){var r=n(3),o=n(23);e.exports=function(e,t){try{o(r,e,t)}catch(n){r[e]=t}return t}},function(e,t,n){var r=n(106),o=Function.toString;"function"!=typeof r.inspectSource&&(r.inspectSource=function(e){return o.call(e)}),e.exports=r.inspectSource},function(e,t,n){var r=n(3),o=n(104),i="__core-js_shared__",a=r[i]||o(i,{});e.exports=a},function(e,t,n){var r=n(36),o=n(106);(e.exports=function(e,t){return o[e]||(o[e]=void 0!==t?t:{})})("versions",[]).push({version:"3.9.0",mode:r?"pure":"global",copyright:"\xa9 2021 Denis Pushkarev (zloirock.ru)"})},function(e,t,n){var r=n(31),o=n(50),i=n(110),a=n(7);e.exports=r("Reflect","ownKeys")||function(e){var t=o.f(a(e)),n=i.f;return n?t.concat(n(e)):t}},function(e,t){e.exports=["constructor","hasOwnProperty","isPrototypeOf","propertyIsEnumerable","toLocaleString","toString","valueOf"]},function(e,t){t.f=Object.getOwnPropertySymbols},function(e,t,n){var r=n(2);e.exports=!!Object.getOwnPropertySymbols&&!r((function(){return!String(Symbol())}))},function(e,t,n){var r=n(9),o=n(73),i=r("iterator"),a=Array.prototype;e.exports=function(e){return void 0!==e&&(o.Array===e||a[i]===e)}},function(e,t,n){var r={};r[n(9)("toStringTag")]="z",e.exports="[object z]"===String(r)},function(e,t,n){"use strict";var r=n(84).charAt,o=n(25),i=n(115),a="String Iterator",u=o.set,l=o.getterFor(a);i(String,"String",(function(e){u(this,{type:a,string:String(e),index:0})}),(function(){var e,t=l(this),n=t.string,o=t.index;return o>=n.length?{value:void 0,done:!0}:(e=r(n,o),t.index+=e.length,{value:e,done:!1})}))},function(e,t,n){"use strict";var r=n(1),o=n(162),i=n(32),a=n(56),u=n(37),l=n(23),c=n(20),s=n(9),f=n(36),d=n(73),p=n(163),h=p.IteratorPrototype,v=p.BUGGY_SAFARI_ITERATORS,y=s("iterator"),g="keys",m="values",b="entries",w=function(){return this};e.exports=function(e,t,n,s,p,x,E){o(n,t,s);var S,k,O,_=function(e){if(e===p&&N)return N;if(!v&&e in P)return P[e];switch(e){case g:case m:case b:return function(){return new n(this,e)}}return function(){return new n(this)}},T=t+" Iterator",C=!1,P=e.prototype,j=P[y]||P["@@iterator"]||p&&P[p],N=!v&&j||_(p),A="Array"==t&&P.entries||j;if(A&&(S=i(A.call(new e)),h!==Object.prototype&&S.next&&(f||i(S)===h||(a?a(S,h):"function"!=typeof S[y]&&l(S,y,w)),u(S,T,!0,!0),f&&(d[T]=w))),p==m&&j&&j.name!==m&&(C=!0,N=function(){return j.call(this)}),f&&!E||P[y]===N||l(P,y,N),d[t]=N,p)if(k={values:_(m),keys:x?N:_(g),entries:_(b)},E)for(O in k)(v||C||!(O in P))&&c(P,O,k[O]);else r({target:t,proto:!0,forced:v||C},k);return k}},function(e,t,n){var r=n(2);e.exports=!r((function(){function e(){}return e.prototype.constructor=null,Object.getPrototypeOf(new e)!==e.prototype}))},function(e,t,n){"use strict";var r=n(14),o=n(51),i=n(12);e.exports=function(e){for(var t=r(this),n=i(t.length),a=arguments.length,u=o(a>1?arguments[1]:void 0,n),l=a>2?arguments[2]:void 0,c=void 0===l?n:o(l,n);c>u;)t[u++]=e;return t}},function(e,t){e.exports=Math.sign||function(e){return 0==(e=+e)||e!=e?e:e<0?-1:1}},function(e,t,n){"use strict";var r=n(29),o=n(22);e.exports="".repeat||function(e){var t=String(o(this)),n="",i=r(e);if(i<0||i==1/0)throw RangeError("Wrong number of repetitions");for(;i>0;(i>>>=1)&&(t+=t))1&i&&(n+=t);return n}},function(e,t,n){var r,o,i,a=n(3),u=n(2),l=n(45),c=n(139),s=n(103),f=n(179),d=n(57),p=a.location,h=a.setImmediate,v=a.clearImmediate,y=a.process,g=a.MessageChannel,m=a.Dispatch,b=0,w={},x="onreadystatechange",E=function(e){if(w.hasOwnProperty(e)){var t=w[e];delete w[e],t()}},S=function(e){return function(){E(e)}},k=function(e){E(e.data)},O=function(e){a.postMessage(e+"",p.protocol+"//"+p.host)};h&&v||(h=function(e){for(var t=[],n=1;arguments.length>n;)t.push(arguments[n++]);return w[++b]=function(){("function"==typeof e?e:Function(e)).apply(void 0,t)},r(b),b},v=function(e){delete w[e]},d?r=function(e){y.nextTick(S(e))}:m&&m.now?r=function(e){m.now(S(e))}:g&&!f?(i=(o=new g).port2,o.port1.onmessage=k,r=l(i.postMessage,i,1)):a.addEventListener&&"function"==typeof postMessage&&!a.importScripts&&p&&"file:"!==p.protocol&&!u(O)?(r=O,a.addEventListener("message",k,!1)):r=x in s("script")?function(e){c.appendChild(s("script")).onreadystatechange=function(){c.removeChild(this),E(e)}}:function(e){setTimeout(S(e),0)}),e.exports={set:h,clear:v}},function(e,t,n){var r=n(6),o=n(43),i=n(9)("match");e.exports=function(e){var t;return r(e)&&(void 0!==(t=e[i])?!!t:"RegExp"==o(e))}},function(e,t,n){"use strict";var r=n(2);function o(e,t){return RegExp(e,t)}t.UNSUPPORTED_Y=r((function(){var e=o("a","y");return e.lastIndex=2,null!=e.exec("abcd")})),t.BROKEN_CARET=r((function(){var e=o("^r","gy");return e.lastIndex=2,null!=e.exec("str")}))},function(e,t,n){var r=n(121);e.exports=function(e){if(r(e))throw TypeError("The method doesn't accept regular expressions");return e}},function(e,t,n){var r=n(9)("match");e.exports=function(e){var t=/./;try{"/./"[e](t)}catch(n){try{return t[r]=!1,"/./"[e](t)}catch(o){}}return!1}},function(e,t,n){"use strict";var r=n(84).charAt;e.exports=function(e,t,n){return t+(n?r(e,t).length:1)}},function(e,t,n){var r=n(2),o=n(91);e.exports=function(e){return r((function(){return!!o[e]()||"\u200b\x85\u180e"!="\u200b\x85\u180e"[e]()||o[e].name!==e}))}},function(e,t,n){var r=n(3),o=n(2),i=n(83),a=n(10).NATIVE_ARRAY_BUFFER_VIEWS,u=r.ArrayBuffer,l=r.Int8Array;e.exports=!a||!o((function(){l(1)}))||!o((function(){new l(-1)}))||!i((function(e){new l,new l(null),new l(1.5),new l(e)}),!0)||o((function(){return 1!==new l(new u(2),1,void 0).length}))},function(e,t,n){"use strict";e.exports=function(e,t,n,r,o,i,a,u){if(!e){var l;if(void 0===t)l=new Error("Minified exception occurred; use the non-minified dev environment for the full error message and additional helpful warnings.");else{var c=[n,r,o,i,a,u],s=0;(l=new Error(t.replace(/%s/g,(function(){return c[s++]})))).name="Invariant Violation"}throw l.framesToPop=1,l}}},function(e,t,n){"use strict";function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;ti;)o.push(arguments[i++]);if(r=t,(p(t)||void 0!==e)&&!ae(e))return d(t)||(t=function(e,t){if("function"==typeof r&&(t=r.call(this,e,t)),!ae(t))return t}),o[1]=t,q.apply(null,o)}});$.prototype[B]||T($.prototype,B,$.prototype.valueOf),M($,z),N[D]=!0},function(e,t){var n;n=function(){return this}();try{n=n||new Function("return this")()}catch(r){"object"===typeof window&&(n=window)}e.exports=n},function(e,t,n){var r=n(11),o=n(2),i=n(103);e.exports=!r&&!o((function(){return 7!=Object.defineProperty(i("div"),"a",{get:function(){return 7}}).a}))},function(e,t,n){var r=n(3),o=n(105),i=r.WeakMap;e.exports="function"===typeof i&&/native code/.test(o(i))},function(e,t,n){var r=n(16),o=n(108),i=n(18),a=n(13);e.exports=function(e,t){for(var n=o(t),u=a.f,l=i.f,c=0;cl;)r(u,n=t[l++])&&(~i(c,n)||c.push(n));return c}},function(e,t,n){var r=n(111);e.exports=r&&!Symbol.sham&&"symbol"==typeof Symbol.iterator},function(e,t,n){var r=n(11),o=n(13),i=n(7),a=n(69);e.exports=r?Object.defineProperties:function(e,t){i(e);for(var n,r=a(t),u=r.length,l=0;u>l;)o.f(e,n=r[l++],t[n]);return e}},function(e,t,n){var r=n(31);e.exports=r("document","documentElement")},function(e,t,n){var r=n(24),o=n(50).f,i={}.toString,a="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[];e.exports.f=function(e){return a&&"[object Window]"==i.call(e)?function(e){try{return o(e)}catch(t){return a.slice()}}(e):o(r(e))}},function(e,t,n){var r=n(9);t.f=r},function(e,t,n){"use strict";var r=n(1),o=n(11),i=n(3),a=n(16),u=n(6),l=n(13).f,c=n(134),s=i.Symbol;if(o&&"function"==typeof s&&(!("description"in s.prototype)||void 0!==s().description)){var f={},d=function(){var e=arguments.length<1||void 0===arguments[0]?void 0:String(arguments[0]),t=this instanceof d?new s(e):void 0===e?s():s(e);return""===e&&(f[t]=!0),t};c(d,s);var p=d.prototype=s.prototype;p.constructor=d;var h=p.toString,v="Symbol(test)"==String(s("test")),y=/^Symbol\((.*)\)[^)]+$/;l(p,"description",{configurable:!0,get:function(){var e=u(this)?this.valueOf():this,t=h.call(e);if(a(f,e))return"";var n=v?t.slice(7,-1):t.replace(y,"$1");return""===n?void 0:n}}),r({global:!0,forced:!0},{Symbol:d})}},function(e,t,n){n(19)("asyncIterator")},function(e,t,n){n(19)("hasInstance")},function(e,t,n){n(19)("isConcatSpreadable")},function(e,t,n){n(19)("iterator")},function(e,t,n){n(19)("match")},function(e,t,n){n(19)("replace")},function(e,t,n){n(19)("search")},function(e,t,n){n(19)("species")},function(e,t,n){n(19)("split")},function(e,t,n){n(19)("toPrimitive")},function(e,t,n){n(19)("toStringTag")},function(e,t,n){n(19)("unscopables")},function(e,t,n){"use strict";var r=n(1),o=n(2),i=n(68),a=n(6),u=n(14),l=n(12),c=n(52),s=n(70),f=n(71),d=n(9),p=n(72),h=d("isConcatSpreadable"),v=9007199254740991,y="Maximum allowed index exceeded",g=p>=51||!o((function(){var e=[];return e[h]=!1,e.concat()[0]!==e})),m=f("concat"),b=function(e){if(!a(e))return!1;var t=e[h];return void 0!==t?!!t:i(e)};r({target:"Array",proto:!0,forced:!g||!m},{concat:function(e){var t,n,r,o,i,a=u(this),f=s(a,0),d=0;for(t=-1,r=arguments.length;tv)throw TypeError(y);for(n=0;n=v)throw TypeError(y);c(f,d++,i)}return f.length=d,f}})},function(e,t,n){var r=n(1),o=n(157);r({target:"Array",stat:!0,forced:!n(83)((function(e){Array.from(e)}))},{from:o})},function(e,t,n){"use strict";var r=n(45),o=n(14),i=n(200),a=n(112),u=n(12),l=n(52),c=n(74);e.exports=function(e){var t,n,s,f,d,p,h=o(e),v="function"==typeof this?this:Array,y=arguments.length,g=y>1?arguments[1]:void 0,m=void 0!==g,b=c(h),w=0;if(m&&(g=r(g,y>2?arguments[2]:void 0,2)),void 0==b||v==Array&&a(b))for(n=new v(t=u(h.length));t>w;w++)p=m?g(h[w],w):h[w],l(n,w,p);else for(d=(f=b.call(h)).next,n=new v;!(s=d.call(f)).done;w++)p=m?i(f,g,[s.value,w],!0):s.value,l(n,w,p);return n.length=w,n}},function(e,t,n){var r=n(7);e.exports=function(e){var t=e.return;if(void 0!==t)return r(t.call(e)).value}},function(e,t,n){var r=n(3);n(37)(r.JSON,"JSON",!0)},function(e,t,n){n(37)(Math,"Math",!0)},function(e,t,n){var r=n(113),o=n(20),i=n(201);r||o(Object.prototype,"toString",i,{unsafe:!0})},function(e,t,n){"use strict";var r=n(163).IteratorPrototype,o=n(44),i=n(42),a=n(37),u=n(73),l=function(){return this};e.exports=function(e,t,n){var c=t+" Iterator";return e.prototype=o(r,{next:i(1,n)}),a(e,c,!1,!0),u[c]=l,e}},function(e,t,n){"use strict";var r,o,i,a=n(2),u=n(32),l=n(23),c=n(16),s=n(9),f=n(36),d=s("iterator"),p=!1;[].keys&&("next"in(i=[].keys())?(o=u(u(i)))!==Object.prototype&&(r=o):p=!0);var h=void 0==r||a((function(){var e={};return r[d].call(e)!==e}));h&&(r={}),f&&!h||c(r,d)||l(r,d,(function(){return this})),e.exports={IteratorPrototype:r,BUGGY_SAFARI_ITERATORS:p}},function(e,t,n){var r=n(6);e.exports=function(e){if(!r(e)&&null!==e)throw TypeError("Can't set "+String(e)+" as a prototype");return e}},function(e,t,n){"use strict";var r=n(206);function o(){}var i=null,a={};function u(e){if("object"!==typeof this)throw new TypeError("Promises must be constructed via new");if("function"!==typeof e)throw new TypeError("Promise constructor's argument is not a function");this._U=0,this._V=0,this._W=null,this._X=null,e!==o&&p(e,this)}function l(e,t){for(;3===e._V;)e=e._W;if(u._Y&&u._Y(e),0===e._V)return 0===e._U?(e._U=1,void(e._X=t)):1===e._U?(e._U=2,void(e._X=[e._X,t])):void e._X.push(t);!function(e,t){r((function(){var n=1===e._V?t.onFulfilled:t.onRejected;if(null!==n){var r=function(e,t){try{return e(t)}catch(n){return i=n,a}}(n,e._W);r===a?s(t.promise,i):c(t.promise,r)}else 1===e._V?c(t.promise,e._W):s(t.promise,e._W)}))}(e,t)}function c(e,t){if(t===e)return s(e,new TypeError("A promise cannot be resolved with itself."));if(t&&("object"===typeof t||"function"===typeof t)){var n=function(e){try{return e.then}catch(t){return i=t,a}}(t);if(n===a)return s(e,i);if(n===e.then&&t instanceof u)return e._V=3,e._W=t,void f(e);if("function"===typeof n)return void p(n.bind(t),e)}e._V=1,e._W=t,f(e)}function s(e,t){e._V=2,e._W=t,u._Z&&u._Z(e,t),f(e)}function f(e){if(1===e._U&&(l(e,e._X),e._X=null),2===e._U){for(var t=0;t2?arguments[2]:void 0,f=a((void 0===s?u:o(s,u))-c,u-l),d=1;for(c0;)c in n?n[l]=n[c]:delete n[l],l+=d,c+=d;return n}},function(e,t,n){"use strict";var r=n(68),o=n(12),i=n(45);e.exports=function e(t,n,a,u,l,c,s,f){for(var d,p=l,h=0,v=!!s&&i(s,f,3);h0&&r(d))p=e(t,n,d,o(d.length),p,c-1)-1;else{if(p>=9007199254740991)throw TypeError("Exceed the acceptable array length");t[p]=d}p++}h++}return p}},function(e,t,n){"use strict";var r=n(24),o=n(29),i=n(12),a=n(53),u=Math.min,l=[].lastIndexOf,c=!!l&&1/[1].lastIndexOf(1,-0)<0,s=a("lastIndexOf"),f=c||!s;e.exports=f?function(e){if(c)return l.apply(this,arguments)||0;var t=r(this),n=i(t.length),a=n-1;for(arguments.length>1&&(a=u(a,o(arguments[1]))),a<0&&(a=n+a);a>=0;a--)if(a in t&&t[a]===e)return a||0;return-1}:l},function(e,t,n){"use strict";var r=n(3),o=n(11),i=n(170),a=n(23),u=n(59),l=n(2),c=n(47),s=n(29),f=n(12),d=n(171),p=n(232),h=n(32),v=n(56),y=n(50).f,g=n(13).f,m=n(117),b=n(37),w=n(25),x=w.get,E=w.set,S="ArrayBuffer",k="DataView",O="Wrong index",_=r.ArrayBuffer,T=_,C=r.DataView,P=C&&C.prototype,j=Object.prototype,N=r.RangeError,A=p.pack,R=p.unpack,L=function(e){return[255&e]},I=function(e){return[255&e,e>>8&255]},M=function(e){return[255&e,e>>8&255,e>>16&255,e>>24&255]},F=function(e){return e[3]<<24|e[2]<<16|e[1]<<8|e[0]},U=function(e){return A(e,23,4)},D=function(e){return A(e,52,8)},z=function(e,t){g(e.prototype,t,{get:function(){return x(this)[t]}})},B=function(e,t,n,r){var o=d(n),i=x(e);if(o+t>i.byteLength)throw N(O);var a=x(i.buffer).bytes,u=o+i.byteOffset,l=a.slice(u,u+t);return r?l:l.reverse()},V=function(e,t,n,r,o,i){var a=d(n),u=x(e);if(a+t>u.byteLength)throw N(O);for(var l=x(u.buffer).bytes,c=a+u.byteOffset,s=r(+o),f=0;fq;)(W=$[q++])in T||a(T,W,_[W]);H.constructor=T}v&&h(P)!==j&&v(P,j);var Q=new C(new T(2)),Y=P.setInt8;Q.setInt8(0,2147483648),Q.setInt8(1,2147483649),!Q.getInt8(0)&&Q.getInt8(1)||u(P,{setInt8:function(e,t){Y.call(this,e,t<<24>>24)},setUint8:function(e,t){Y.call(this,e,t<<24>>24)}},{unsafe:!0})}else T=function(e){c(this,T,S);var t=d(e);E(this,{bytes:m.call(new Array(t),0),byteLength:t}),o||(this.byteLength=t)},C=function(e,t,n){c(this,C,k),c(e,T,k);var r=x(e).byteLength,i=s(t);if(i<0||i>r)throw N("Wrong offset");if(i+(n=void 0===n?r-i:f(n))>r)throw N("Wrong length");E(this,{buffer:e,byteLength:n,byteOffset:i}),o||(this.buffer=e,this.byteLength=n,this.byteOffset=i)},o&&(z(T,"byteLength"),z(C,"buffer"),z(C,"byteLength"),z(C,"byteOffset")),u(C.prototype,{getInt8:function(e){return B(this,1,e)[0]<<24>>24},getUint8:function(e){return B(this,1,e)[0]},getInt16:function(e){var t=B(this,2,e,arguments.length>1?arguments[1]:void 0);return(t[1]<<8|t[0])<<16>>16},getUint16:function(e){var t=B(this,2,e,arguments.length>1?arguments[1]:void 0);return t[1]<<8|t[0]},getInt32:function(e){return F(B(this,4,e,arguments.length>1?arguments[1]:void 0))},getUint32:function(e){return F(B(this,4,e,arguments.length>1?arguments[1]:void 0))>>>0},getFloat32:function(e){return R(B(this,4,e,arguments.length>1?arguments[1]:void 0),23)},getFloat64:function(e){return R(B(this,8,e,arguments.length>1?arguments[1]:void 0),52)},setInt8:function(e,t){V(this,1,e,L,t)},setUint8:function(e,t){V(this,1,e,L,t)},setInt16:function(e,t){V(this,2,e,I,t,arguments.length>2?arguments[2]:void 0)},setUint16:function(e,t){V(this,2,e,I,t,arguments.length>2?arguments[2]:void 0)},setInt32:function(e,t){V(this,4,e,M,t,arguments.length>2?arguments[2]:void 0)},setUint32:function(e,t){V(this,4,e,M,t,arguments.length>2?arguments[2]:void 0)},setFloat32:function(e,t){V(this,4,e,U,t,arguments.length>2?arguments[2]:void 0)},setFloat64:function(e,t){V(this,8,e,D,t,arguments.length>2?arguments[2]:void 0)}});b(T,S),b(C,k),e.exports={ArrayBuffer:T,DataView:C}},function(e,t){e.exports="undefined"!==typeof ArrayBuffer&&"undefined"!==typeof DataView},function(e,t,n){var r=n(29),o=n(12);e.exports=function(e){if(void 0===e)return 0;var t=r(e),n=o(t);if(t!==n)throw RangeError("Wrong length or index");return n}},function(e,t,n){"use strict";var r=n(13).f,o=n(44),i=n(59),a=n(45),u=n(47),l=n(77),c=n(115),s=n(58),f=n(11),d=n(54).fastKey,p=n(25),h=p.set,v=p.getterFor;e.exports={getConstructor:function(e,t,n,c){var s=e((function(e,r){u(e,s,t),h(e,{type:t,index:o(null),first:void 0,last:void 0,size:0}),f||(e.size=0),void 0!=r&&l(r,e[c],{that:e,AS_ENTRIES:n})})),p=v(t),y=function(e,t,n){var r,o,i=p(e),a=g(e,t);return a?a.value=n:(i.last=a={index:o=d(t,!0),key:t,value:n,previous:r=i.last,next:void 0,removed:!1},i.first||(i.first=a),r&&(r.next=a),f?i.size++:e.size++,"F"!==o&&(i.index[o]=a)),e},g=function(e,t){var n,r=p(e),o=d(t);if("F"!==o)return r.index[o];for(n=r.first;n;n=n.next)if(n.key==t)return n};return i(s.prototype,{clear:function(){for(var e=p(this),t=e.index,n=e.first;n;)n.removed=!0,n.previous&&(n.previous=n.previous.next=void 0),delete t[n.index],n=n.next;e.first=e.last=void 0,f?e.size=0:this.size=0},delete:function(e){var t=this,n=p(t),r=g(t,e);if(r){var o=r.next,i=r.previous;delete n.index[r.index],r.removed=!0,i&&(i.next=o),o&&(o.previous=i),n.first==r&&(n.first=o),n.last==r&&(n.last=i),f?n.size--:t.size--}return!!r},forEach:function(e){for(var t,n=p(this),r=a(e,arguments.length>1?arguments[1]:void 0,3);t=t?t.next:n.first;)for(r(t.value,t.key,this);t&&t.removed;)t=t.previous},has:function(e){return!!g(this,e)}}),i(s.prototype,n?{get:function(e){var t=g(this,e);return t&&t.value},set:function(e,t){return y(this,0===e?0:e,t)}}:{add:function(e){return y(this,e=0===e?0:e,e)}}),f&&r(s.prototype,"size",{get:function(){return p(this).size}}),s},setStrong:function(e,t,n){var r=t+" Iterator",o=v(t),i=v(r);c(e,t,(function(e,t){h(this,{type:r,target:e,state:o(e),kind:t,last:void 0})}),(function(){for(var e=i(this),t=e.kind,n=e.last;n&&n.removed;)n=n.previous;return e.target&&(e.last=n=n?n.next:e.state.first)?"keys"==t?{value:n.key,done:!1}:"values"==t?{value:n.value,done:!1}:{value:[n.key,n.value],done:!1}:(e.target=void 0,{value:void 0,done:!0})}),n?"entries":"values",!n,!0),s(t)}}},function(e,t){var n=Math.log;e.exports=Math.log1p||function(e){return(e=+e)>-1e-8&&e<1e-8?e-e*e/2:n(1+e)}},function(e,t,n){var r=n(6),o=Math.floor;e.exports=function(e){return!r(e)&&isFinite(e)&&o(e)===e}},function(e,t,n){"use strict";var r=n(11),o=n(2),i=n(69),a=n(110),u=n(80),l=n(14),c=n(63),s=Object.assign,f=Object.defineProperty;e.exports=!s||o((function(){if(r&&1!==s({b:1},s(f({},"a",{enumerable:!0,get:function(){f(this,"b",{value:3,enumerable:!1})}}),{b:2})).b)return!0;var e={},t={},n=Symbol(),o="abcdefghijklmnopqrst";return e[n]=7,o.split("").forEach((function(e){t[e]=e})),7!=s({},e)[n]||i(s({},t)).join("")!=o}))?function(e,t){for(var n=l(e),o=arguments.length,s=1,f=a.f,d=u.f;o>s;)for(var p,h=c(arguments[s++]),v=f?i(h).concat(f(h)):i(h),y=v.length,g=0;y>g;)p=v[g++],r&&!d.call(h,p)||(n[p]=h[p]);return n}:s},function(e,t,n){var r=n(11),o=n(69),i=n(24),a=n(80).f,u=function(e){return function(t){for(var n,u=i(t),l=o(u),c=l.length,s=0,f=[];c>s;)n=l[s++],r&&!a.call(u,n)||f.push(e?[n,u[n]]:u[n]);return f}};e.exports={entries:u(!0),values:u(!1)}},function(e,t){e.exports=Object.is||function(e,t){return e===t?0!==e||1/e===1/t:e!=e&&t!=t}},function(e,t,n){var r=n(3);e.exports=r.Promise},function(e,t,n){var r=n(82);e.exports=/(iphone|ipod|ipad).*applewebkit/i.test(r)},function(e,t,n){var r,o,i,a,u,l,c,s,f=n(3),d=n(18).f,p=n(120).set,h=n(179),v=n(292),y=n(57),g=f.MutationObserver||f.WebKitMutationObserver,m=f.document,b=f.process,w=f.Promise,x=d(f,"queueMicrotask"),E=x&&x.value;E||(r=function(){var e,t;for(y&&(e=b.domain)&&e.exit();o;){t=o.fn,o=o.next;try{t()}catch(n){throw o?a():i=void 0,n}}i=void 0,e&&e.enter()},h||y||v||!g||!m?w&&w.resolve?(c=w.resolve(void 0),s=c.then,a=function(){s.call(c,r)}):a=y?function(){b.nextTick(r)}:function(){p.call(f,r)}:(u=!0,l=m.createTextNode(""),new g(r).observe(l,{characterData:!0}),a=function(){l.data=u=!u})),e.exports=E||function(e){var t={fn:e,next:void 0};i&&(i.next=t),o||(o=t,a()),i=t}},function(e,t,n){var r=n(7),o=n(6),i=n(182);e.exports=function(e,t){if(r(e),o(t)&&t.constructor===e)return t;var n=i.f(e);return(0,n.resolve)(t),n.promise}},function(e,t,n){"use strict";var r=n(30),o=function(e){var t,n;this.promise=new e((function(e,r){if(void 0!==t||void 0!==n)throw TypeError("Bad Promise constructor");t=e,n=r})),this.resolve=r(t),this.reject=r(n)};e.exports.f=function(e){return new o(e)}},function(e,t,n){"use strict";var r=n(1),o=n(94);r({target:"RegExp",proto:!0,forced:/./.exec!==o},{exec:o})},function(e,t,n){var r=n(12),o=n(119),i=n(22),a=Math.ceil,u=function(e){return function(t,n,u){var l,c,s=String(i(t)),f=s.length,d=void 0===u?" ":String(u),p=r(n);return p<=f||""==d?s:(l=p-f,(c=o.call(d,a(l/d.length))).length>l&&(c=c.slice(0,l)),e?s+c:c+s)}};e.exports={start:u(!1),end:u(!0)}},function(e,t,n){var r=n(82);e.exports=/Version\/10\.\d+(\.\d+)?( Mobile\/\w+)? Safari\//.test(r)},function(e,t,n){var r=n(345);e.exports=function(e,t){var n=r(e);if(n%t)throw RangeError("Wrong offset");return n}},function(e,t,n){var r=n(14),o=n(12),i=n(74),a=n(112),u=n(45),l=n(10).aTypedArrayConstructor;e.exports=function(e){var t,n,c,s,f,d,p=r(e),h=arguments.length,v=h>1?arguments[1]:void 0,y=void 0!==v,g=i(p);if(void 0!=g&&!a(g))for(d=(f=g.call(p)).next,p=[];!(s=d.call(f)).done;)p.push(s.value);for(y&&h>2&&(v=u(v,arguments[2],2)),n=o(p.length),c=new(l(this))(n),t=0;n>t;t++)c[t]=y?v(p[t],t):p[t];return c}},function(e,t,n){"use strict";var r=n(59),o=n(54).getWeakData,i=n(7),a=n(6),u=n(47),l=n(77),c=n(21),s=n(16),f=n(25),d=f.set,p=f.getterFor,h=c.find,v=c.findIndex,y=0,g=function(e){return e.frozen||(e.frozen=new m)},m=function(){this.entries=[]},b=function(e,t){return h(e.entries,(function(e){return e[0]===t}))};m.prototype={get:function(e){var t=b(this,e);if(t)return t[1]},has:function(e){return!!b(this,e)},set:function(e,t){var n=b(this,e);n?n[1]=t:this.entries.push([e,t])},delete:function(e){var t=v(this.entries,(function(t){return t[0]===e}));return~t&&this.entries.splice(t,1),!!~t}},e.exports={getConstructor:function(e,t,n,c){var f=e((function(e,r){u(e,f,t),d(e,{type:t,id:y++,frozen:void 0}),void 0!=r&&l(r,e[c],{that:e,AS_ENTRIES:n})})),h=p(t),v=function(e,t,n){var r=h(e),a=o(i(t),!0);return!0===a?g(r).set(t,n):a[r.id]=n,e};return r(f.prototype,{delete:function(e){var t=h(this);if(!a(e))return!1;var n=o(e);return!0===n?g(t).delete(e):n&&s(n,t.id)&&delete n[t.id]},has:function(e){var t=h(this);if(!a(e))return!1;var n=o(e);return!0===n?g(t).has(e):n&&s(n,t.id)}}),r(f.prototype,n?{get:function(e){var t=h(this);if(a(e)){var n=o(e);return!0===n?g(t).get(e):n?n[t.id]:void 0}},set:function(e,t){return v(this,e,t)}}:{add:function(e){return v(this,e,!0)}}),f}}},function(e,t){e.exports={CSSRuleList:0,CSSStyleDeclaration:0,CSSValueList:0,ClientRectList:0,DOMRectList:0,DOMStringList:0,DOMTokenList:1,DataTransferItemList:0,FileList:0,HTMLAllCollection:0,HTMLCollection:0,HTMLFormElement:0,HTMLSelectElement:0,MediaList:0,MimeTypeArray:0,NamedNodeMap:0,NodeList:1,PaintRequestList:0,Plugin:0,PluginArray:0,SVGLengthList:0,SVGNumberList:0,SVGPathSegList:0,SVGPointList:0,SVGStringList:0,SVGTransformList:0,SourceBufferList:0,StyleSheetList:0,TextTrackCueList:0,TextTrackList:0,TouchList:0}},function(e,t,n){var r=n(2),o=n(9),i=n(36),a=o("iterator");e.exports=!r((function(){var e=new URL("b?a=1&b=2&c=3","http://a"),t=e.searchParams,n="";return e.pathname="c%20d",t.forEach((function(e,r){t.delete("b"),n+=r+e})),i&&!e.toJSON||!t.sort||"http://a/c%20d?a=1&c=3"!==e.href||"3"!==t.get("c")||"a=1"!==String(new URLSearchParams("?a=1"))||!t[a]||"a"!==new URL("https://a@b").username||"b"!==new URLSearchParams(new URLSearchParams("a=b")).get("a")||"xn--e1aybc"!==new URL("http://\u0442\u0435\u0441\u0442").host||"#%D0%B1"!==new URL("http://a#\u0431").hash||"a1c3"!==n||"x"!==new URL("http://x",void 0).host}))},function(e,t,n){"use strict";n(86);var r=n(1),o=n(31),i=n(190),a=n(20),u=n(59),l=n(37),c=n(162),s=n(25),f=n(47),d=n(16),p=n(45),h=n(75),v=n(7),y=n(6),g=n(44),m=n(42),b=n(389),w=n(74),x=n(9),E=o("fetch"),S=o("Headers"),k=x("iterator"),O="URLSearchParams",_="URLSearchParamsIterator",T=s.set,C=s.getterFor(O),P=s.getterFor(_),j=/\+/g,N=Array(4),A=function(e){return N[e-1]||(N[e-1]=RegExp("((?:%[\\da-f]{2}){"+e+"})","gi"))},R=function(e){try{return decodeURIComponent(e)}catch(t){return e}},L=function(e){var t=e.replace(j," "),n=4;try{return decodeURIComponent(t)}catch(r){for(;n;)t=t.replace(A(n--),R);return t}},I=/[!'()~]|%20/g,M={"!":"%21","'":"%27","(":"%28",")":"%29","~":"%7E","%20":"+"},F=function(e){return M[e]},U=function(e){return encodeURIComponent(e).replace(I,F)},D=function(e,t){if(t)for(var n,r,o=t.split("&"),i=0;i0?arguments[0]:void 0,s=this,p=[];if(T(s,{type:O,entries:p,updateURL:function(){},updateSearchParams:z}),void 0!==c)if(y(c))if("function"===typeof(e=w(c)))for(n=(t=e.call(c)).next;!(r=n.call(t)).done;){if((a=(i=(o=b(v(r.value))).next).call(o)).done||(u=i.call(o)).done||!i.call(o).done)throw TypeError("Expected sequence with length 2");p.push({key:a.value+"",value:u.value+""})}else for(l in c)d(c,l)&&p.push({key:l,value:c[l]+""});else D(p,"string"===typeof c?"?"===c.charAt(0)?c.slice(1):c:c+"")},H=W.prototype;u(H,{append:function(e,t){B(arguments.length,2);var n=C(this);n.entries.push({key:e+"",value:t+""}),n.updateURL()},delete:function(e){B(arguments.length,1);for(var t=C(this),n=t.entries,r=e+"",o=0;oe.key){o.splice(t,0,e);break}t===n&&o.push(e)}r.updateURL()},forEach:function(e){for(var t,n=C(this).entries,r=p(e,arguments.length>1?arguments[1]:void 0,3),o=0;o1&&(y(t=arguments[1])&&(n=t.body,h(n)===O&&((r=t.headers?new S(t.headers):new S).has("content-type")||r.set("content-type","application/x-www-form-urlencoded;charset=UTF-8"),t=g(t,{body:m(0,String(n)),headers:m(0,r)}))),o.push(t)),E.apply(this,o)}}),e.exports={URLSearchParams:W,getState:C}},function(e,t,n){var r=function(e){"use strict";var t,n=Object.prototype,r=n.hasOwnProperty,o="function"===typeof Symbol?Symbol:{},i=o.iterator||"@@iterator",a=o.asyncIterator||"@@asyncIterator",u=o.toStringTag||"@@toStringTag";function l(e,t,n){return Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}),e[t]}try{l({},"")}catch(N){l=function(e,t,n){return e[t]=n}}function c(e,t,n,r){var o=t&&t.prototype instanceof y?t:y,i=Object.create(o.prototype),a=new C(r||[]);return i._invoke=function(e,t,n){var r=f;return function(o,i){if(r===p)throw new Error("Generator is already running");if(r===h){if("throw"===o)throw i;return j()}for(n.method=o,n.arg=i;;){var a=n.delegate;if(a){var u=O(a,n);if(u){if(u===v)continue;return u}}if("next"===n.method)n.sent=n._sent=n.arg;else if("throw"===n.method){if(r===f)throw r=h,n.arg;n.dispatchException(n.arg)}else"return"===n.method&&n.abrupt("return",n.arg);r=p;var l=s(e,t,n);if("normal"===l.type){if(r=n.done?h:d,l.arg===v)continue;return{value:l.arg,done:n.done}}"throw"===l.type&&(r=h,n.method="throw",n.arg=l.arg)}}}(e,n,a),i}function s(e,t,n){try{return{type:"normal",arg:e.call(t,n)}}catch(N){return{type:"throw",arg:N}}}e.wrap=c;var f="suspendedStart",d="suspendedYield",p="executing",h="completed",v={};function y(){}function g(){}function m(){}var b={};b[i]=function(){return this};var w=Object.getPrototypeOf,x=w&&w(w(P([])));x&&x!==n&&r.call(x,i)&&(b=x);var E=m.prototype=y.prototype=Object.create(b);function S(e){["next","throw","return"].forEach((function(t){l(e,t,(function(e){return this._invoke(t,e)}))}))}function k(e,t){function n(o,i,a,u){var l=s(e[o],e,i);if("throw"!==l.type){var c=l.arg,f=c.value;return f&&"object"===typeof f&&r.call(f,"__await")?t.resolve(f.__await).then((function(e){n("next",e,a,u)}),(function(e){n("throw",e,a,u)})):t.resolve(f).then((function(e){c.value=e,a(c)}),(function(e){return n("throw",e,a,u)}))}u(l.arg)}var o;this._invoke=function(e,r){function i(){return new t((function(t,o){n(e,r,t,o)}))}return o=o?o.then(i,i):i()}}function O(e,n){var r=e.iterator[n.method];if(r===t){if(n.delegate=null,"throw"===n.method){if(e.iterator.return&&(n.method="return",n.arg=t,O(e,n),"throw"===n.method))return v;n.method="throw",n.arg=new TypeError("The iterator does not provide a 'throw' method")}return v}var o=s(r,e.iterator,n.arg);if("throw"===o.type)return n.method="throw",n.arg=o.arg,n.delegate=null,v;var i=o.arg;return i?i.done?(n[e.resultName]=i.value,n.next=e.nextLoc,"return"!==n.method&&(n.method="next",n.arg=t),n.delegate=null,v):i:(n.method="throw",n.arg=new TypeError("iterator result is not an object"),n.delegate=null,v)}function _(e){var t={tryLoc:e[0]};1 in e&&(t.catchLoc=e[1]),2 in e&&(t.finallyLoc=e[2],t.afterLoc=e[3]),this.tryEntries.push(t)}function T(e){var t=e.completion||{};t.type="normal",delete t.arg,e.completion=t}function C(e){this.tryEntries=[{tryLoc:"root"}],e.forEach(_,this),this.reset(!0)}function P(e){if(e){var n=e[i];if(n)return n.call(e);if("function"===typeof e.next)return e;if(!isNaN(e.length)){var o=-1,a=function n(){for(;++o=0;--i){var a=this.tryEntries[i],u=a.completion;if("root"===a.tryLoc)return o("end");if(a.tryLoc<=this.prev){var l=r.call(a,"catchLoc"),c=r.call(a,"finallyLoc");if(l&&c){if(this.prev=0;--n){var o=this.tryEntries[n];if(o.tryLoc<=this.prev&&r.call(o,"finallyLoc")&&this.prev=0;--t){var n=this.tryEntries[t];if(n.finallyLoc===e)return this.complete(n.completion,n.afterLoc),T(n),v}},catch:function(e){for(var t=this.tryEntries.length-1;t>=0;--t){var n=this.tryEntries[t];if(n.tryLoc===e){var r=n.completion;if("throw"===r.type){var o=r.arg;T(n)}return o}}throw new Error("illegal catch attempt")},delegateYield:function(e,n,r){return this.delegate={iterator:P(e),resultName:n,nextLoc:r},"next"===this.method&&(this.arg=t),v}},e}(e.exports);try{regeneratorRuntime=r}catch(o){Function("r","regeneratorRuntime = r")(r)}},function(e,t,n){"use strict";function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}n.d(t,"a",(function(){return r}))},function(e,t,n){"use strict";function r(e,t){for(var n=0;n1024){for(var t=0,n=o.length-i;t-1};function f(e){if("string"!==typeof e&&(e=String(e)),/[^a-z0-9\-#$%&'*+.^_`|~!]/i.test(e)||""===e)throw new TypeError('Invalid character in header field name: "'+e+'"');return e.toLowerCase()}function d(e){return"string"!==typeof e&&(e=String(e)),e}function p(e){var t={next:function(){var t=e.shift();return{done:void 0===t,value:t}}};return i&&(t[Symbol.iterator]=function(){return t}),t}function h(e){this.map={},e instanceof h?e.forEach((function(e,t){this.append(t,e)}),this):Array.isArray(e)?e.forEach((function(e){this.append(e[0],e[1])}),this):e&&Object.getOwnPropertyNames(e).forEach((function(t){this.append(t,e[t])}),this)}function v(e){if(e.bodyUsed)return Promise.reject(new TypeError("Already read"));e.bodyUsed=!0}function y(e){return new Promise((function(t,n){e.onload=function(){t(e.result)},e.onerror=function(){n(e.error)}}))}function g(e){var t=new FileReader,n=y(t);return t.readAsArrayBuffer(e),n}function m(e){if(e.slice)return e.slice(0);var t=new Uint8Array(e.byteLength);return t.set(new Uint8Array(e)),t.buffer}function b(){return this.bodyUsed=!1,this._initBody=function(e){var t;this.bodyUsed=this.bodyUsed,this._bodyInit=e,e?"string"===typeof e?this._bodyText=e:a&&Blob.prototype.isPrototypeOf(e)?this._bodyBlob=e:u&&FormData.prototype.isPrototypeOf(e)?this._bodyFormData=e:o&&URLSearchParams.prototype.isPrototypeOf(e)?this._bodyText=e.toString():l&&a&&((t=e)&&DataView.prototype.isPrototypeOf(t))?(this._bodyArrayBuffer=m(e.buffer),this._bodyInit=new Blob([this._bodyArrayBuffer])):l&&(ArrayBuffer.prototype.isPrototypeOf(e)||s(e))?this._bodyArrayBuffer=m(e):this._bodyText=e=Object.prototype.toString.call(e):this._bodyText="";var n=this.headers.get("content-type");n?n.indexOf("json")>=0&&"string"!==typeof this._bodyInit&&(this._bodyInit=this._bodyText):"string"===typeof e?this.headers.set("content-type","text/plain;charset=UTF-8"):this._bodyBlob&&this._bodyBlob.type?this.headers.set("content-type",this._bodyBlob.type):o&&URLSearchParams.prototype.isPrototypeOf(e)&&this.headers.set("content-type","application/x-www-form-urlencoded;charset=UTF-8")},a&&(this.blob=function(){var e=v(this);if(e)return e;if(this._bodyBlob)return Promise.resolve(this._bodyBlob);if(this._bodyArrayBuffer)return Promise.resolve(new Blob([this._bodyArrayBuffer]));if(this._bodyFormData)throw new Error("could not read FormData body as blob");return Promise.resolve(new Blob([this._bodyText]))},this.arrayBuffer=function(){if(this._bodyArrayBuffer){var e=v(this);return e||(ArrayBuffer.isView(this._bodyArrayBuffer)?Promise.resolve(this._bodyArrayBuffer.buffer.slice(this._bodyArrayBuffer.byteOffset,this._bodyArrayBuffer.byteOffset+this._bodyArrayBuffer.byteLength)):Promise.resolve(this._bodyArrayBuffer))}return this.blob().then(g)}),this.text=function(){var e=v(this);if(e)return e;if(this._bodyBlob)return function(e){var t=new FileReader,n=y(t);return t.readAsText(e),n}(this._bodyBlob);if(this._bodyArrayBuffer)return Promise.resolve(function(e){for(var t=new Uint8Array(e),n=new Array(t.length),r=0;r-1?t:e}(t.method||this.method||"GET"),this.mode=t.mode||this.mode||null,this.signal=t.signal||this.signal,this.referrer=null,("GET"===this.method||"HEAD"===this.method)&&n)throw new TypeError("Body not allowed for GET or HEAD requests");if(this._initBody(n),("GET"===this.method||"HEAD"===this.method)&&("no-store"===t.cache||"no-cache"===t.cache)){var r=/([?&])_=[^&]*/;if(r.test(this.url))this.url=this.url.replace(r,"$1_="+(new Date).getTime());else{this.url+=(/\?/.test(this.url)?"&":"?")+"_="+(new Date).getTime()}}}function E(e){var t=new FormData;return e.trim().split("&").forEach((function(e){if(e){var n=e.split("="),r=n.shift().replace(/\+/g," "),o=n.join("=").replace(/\+/g," ");t.append(decodeURIComponent(r),decodeURIComponent(o))}})),t}function S(e){var t=new h;return e.replace(/\r?\n[\t ]+/g," ").split("\r").map((function(e){return 0===e.indexOf("\n")?e.substr(1,e.length):e})).forEach((function(e){var n=e.split(":"),r=n.shift().trim();if(r){var o=n.join(":").trim();t.append(r,o)}})),t}function k(e,t){if(!(this instanceof k))throw new TypeError('Please use the "new" operator, this DOM object constructor cannot be called as a function.');t||(t={}),this.type="default",this.status=void 0===t.status?200:t.status,this.ok=this.status>=200&&this.status<300,this.statusText=void 0===t.statusText?"":""+t.statusText,this.headers=new h(t.headers),this.url=t.url||"",this._initBody(e)}x.prototype.clone=function(){return new x(this,{body:this._bodyInit})},b.call(x.prototype),b.call(k.prototype),k.prototype.clone=function(){return new k(this._bodyInit,{status:this.status,statusText:this.statusText,headers:new h(this.headers),url:this.url})},k.error=function(){var e=new k(null,{status:0,statusText:""});return e.type="error",e};var O=[301,302,303,307,308];k.redirect=function(e,t){if(-1===O.indexOf(t))throw new RangeError("Invalid status code");return new k(null,{status:t,headers:{location:e}})};var _=r.DOMException;try{new _}catch(C){(_=function(e,t){this.message=e,this.name=t;var n=Error(e);this.stack=n.stack}).prototype=Object.create(Error.prototype),_.prototype.constructor=_}function T(e,t){return new Promise((function(n,o){var i=new x(e,t);if(i.signal&&i.signal.aborted)return o(new _("Aborted","AbortError"));var u=new XMLHttpRequest;function c(){u.abort()}u.onload=function(){var e={status:u.status,statusText:u.statusText,headers:S(u.getAllResponseHeaders()||"")};e.url="responseURL"in u?u.responseURL:e.headers.get("X-Request-URL");var t="response"in u?u.response:u.responseText;setTimeout((function(){n(new k(t,e))}),0)},u.onerror=function(){setTimeout((function(){o(new TypeError("Network request failed"))}),0)},u.ontimeout=function(){setTimeout((function(){o(new TypeError("Network request failed"))}),0)},u.onabort=function(){setTimeout((function(){o(new _("Aborted","AbortError"))}),0)},u.open(i.method,function(e){try{return""===e&&r.location.href?r.location.href:e}catch(t){return e}}(i.url),!0),"include"===i.credentials?u.withCredentials=!0:"omit"===i.credentials&&(u.withCredentials=!1),"responseType"in u&&(a?u.responseType="blob":l&&i.headers.get("Content-Type")&&-1!==i.headers.get("Content-Type").indexOf("application/octet-stream")&&(u.responseType="arraybuffer")),!t||"object"!==typeof t.headers||t.headers instanceof h?i.headers.forEach((function(e,t){u.setRequestHeader(t,e)})):Object.getOwnPropertyNames(t.headers).forEach((function(e){u.setRequestHeader(e,d(t.headers[e]))})),i.signal&&(i.signal.addEventListener("abort",c),u.onreadystatechange=function(){4===u.readyState&&i.signal.removeEventListener("abort",c)}),u.send("undefined"===typeof i._bodyInit?null:i._bodyInit)}))}T.polyfill=!0,r.fetch||(r.fetch=T,r.Headers=h,r.Request=x,r.Response=k)},function(e,t,n){"use strict";n(130),n(142),n(143),n(144),n(145),n(146),n(147),n(148),n(149),n(150),n(151),n(152),n(153),n(154),n(155),n(210),n(211),n(212),n(213),n(214),n(215),n(216),n(156),n(217),n(218),n(86),n(219),n(220),n(221),n(222),n(223),n(224),n(225),n(226),n(227),n(228),n(229),n(230),n(231),n(233),n(235),n(236),n(159),n(237),n(238),n(239),n(240),n(241),n(242),n(243),n(244),n(245),n(247),n(248),n(249),n(250),n(251),n(252),n(253),n(254),n(160),n(255),n(256),n(257),n(258),n(260),n(261),n(262),n(263),n(264),n(265),n(267),n(269),n(271),n(272),n(273),n(274),n(275),n(276),n(277),n(278),n(279),n(280),n(281),n(282),n(283),n(284),n(285),n(286),n(287),n(288),n(289),n(161),n(290),n(291),n(295),n(296),n(297),n(299),n(300),n(301),n(302),n(303),n(304),n(305),n(306),n(307),n(308),n(309),n(310),n(183),n(311),n(312),n(313),n(314),n(315),n(316),n(317),n(114),n(318),n(319),n(320),n(321),n(322),n(323),n(325),n(326),n(327),n(328),n(329),n(330),n(331),n(332),n(333),n(334),n(335),n(336),n(337),n(338),n(339),n(340),n(341),n(342),n(343),n(344),n(346),n(347),n(348),n(349),n(350),n(351),n(352),n(353),n(354),n(355),n(356),n(357),n(359),n(360),n(361),n(362),n(363),n(364),n(365),n(366),n(367),n(368),n(369),n(370),n(371),n(372),n(373),n(374),n(375),n(376),n(377),n(378),n(379),n(380),n(381),n(382),n(384),n(385),n(386),n(387),n(390),n(191),n(192)},function(e,t,n){var r=n(1),o=n(166),i=n(46);r({target:"Array",proto:!0},{copyWithin:o}),i("copyWithin")},function(e,t,n){var r=n(1),o=n(117),i=n(46);r({target:"Array",proto:!0},{fill:o}),i("fill")},function(e,t,n){"use strict";var r=n(1),o=n(21).filter;r({target:"Array",proto:!0,forced:!n(71)("filter")},{filter:function(e){return o(this,e,arguments.length>1?arguments[1]:void 0)}})},function(e,t,n){"use strict";var r=n(1),o=n(21).find,i=n(46),a="find",u=!0;a in[]&&Array(1).find((function(){u=!1})),r({target:"Array",proto:!0,forced:u},{find:function(e){return o(this,e,arguments.length>1?arguments[1]:void 0)}}),i(a)},function(e,t,n){"use strict";var r=n(1),o=n(21).findIndex,i=n(46),a="findIndex",u=!0;a in[]&&Array(1).findIndex((function(){u=!1})),r({target:"Array",proto:!0,forced:u},{findIndex:function(e){return o(this,e,arguments.length>1?arguments[1]:void 0)}}),i(a)},function(e,t,n){"use strict";var r=n(1),o=n(167),i=n(14),a=n(12),u=n(29),l=n(70);r({target:"Array",proto:!0},{flat:function(){var e=arguments.length?arguments[0]:void 0,t=i(this),n=a(t.length),r=l(t,0);return r.length=o(r,t,t,n,0,void 0===e?1:u(e)),r}})},function(e,t,n){"use strict";var r=n(1),o=n(167),i=n(14),a=n(12),u=n(30),l=n(70);r({target:"Array",proto:!0},{flatMap:function(e){var t,n=i(this),r=a(n.length);return u(e),(t=l(n,0)).length=o(t,n,n,r,0,1,e,arguments.length>1?arguments[1]:void 0),t}})},function(e,t,n){"use strict";var r=n(1),o=n(66).includes,i=n(46);r({target:"Array",proto:!0},{includes:function(e){return o(this,e,arguments.length>1?arguments[1]:void 0)}}),i("includes")},function(e,t,n){"use strict";var r=n(1),o=n(66).indexOf,i=n(53),a=[].indexOf,u=!!a&&1/[1].indexOf(1,-0)<0,l=i("indexOf");r({target:"Array",proto:!0,forced:u||!l},{indexOf:function(e){return u?a.apply(this,arguments)||0:o(this,e,arguments.length>1?arguments[1]:void 0)}})},function(e,t,n){"use strict";var r=n(1),o=n(63),i=n(24),a=n(53),u=[].join,l=o!=Object,c=a("join",",");r({target:"Array",proto:!0,forced:l||!c},{join:function(e){return u.call(i(this),void 0===e?",":e)}})},function(e,t,n){var r=n(1),o=n(168);r({target:"Array",proto:!0,forced:o!==[].lastIndexOf},{lastIndexOf:o})},function(e,t,n){"use strict";var r=n(1),o=n(21).map;r({target:"Array",proto:!0,forced:!n(71)("map")},{map:function(e){return o(this,e,arguments.length>1?arguments[1]:void 0)}})},function(e,t,n){"use strict";var r=n(1),o=n(2),i=n(52);r({target:"Array",stat:!0,forced:o((function(){function e(){}return!(Array.of.call(e)instanceof e)}))},{of:function(){for(var e=0,t=arguments.length,n=new("function"==typeof this?this:Array)(t);t>e;)i(n,e,arguments[e++]);return n.length=t,n}})},function(e,t,n){"use strict";var r=n(1),o=n(87).left,i=n(53),a=n(72),u=n(57);r({target:"Array",proto:!0,forced:!i("reduce")||!u&&a>79&&a<83},{reduce:function(e){return o(this,e,arguments.length,arguments.length>1?arguments[1]:void 0)}})},function(e,t,n){"use strict";var r=n(1),o=n(87).right,i=n(53),a=n(72),u=n(57);r({target:"Array",proto:!0,forced:!i("reduceRight")||!u&&a>79&&a<83},{reduceRight:function(e){return o(this,e,arguments.length,arguments.length>1?arguments[1]:void 0)}})},function(e,t,n){"use strict";var r=n(1),o=n(6),i=n(68),a=n(51),u=n(12),l=n(24),c=n(52),s=n(9),f=n(71)("slice"),d=s("species"),p=[].slice,h=Math.max;r({target:"Array",proto:!0,forced:!f},{slice:function(e,t){var n,r,s,f=l(this),v=u(f.length),y=a(e,v),g=a(void 0===t?v:t,v);if(i(f)&&("function"!=typeof(n=f.constructor)||n!==Array&&!i(n.prototype)?o(n)&&null===(n=n[d])&&(n=void 0):n=void 0,n===Array||void 0===n))return p.call(f,y,g);for(r=new(void 0===n?Array:n)(h(g-y,0)),s=0;yp)throw TypeError(h);for(s=l(m,r),v=0;vb-r+n;v--)delete m[v-1]}else if(n>r)for(v=b-r;v>w;v--)g=v+n-1,(y=v+r-1)in m?m[g]=m[y]:delete m[g];for(v=0;v>1,v=23===t?r(2,-24)-r(2,-77):0,y=e<0||0===e&&1/e<0?1:0,g=0;for((e=n(e))!=e||e===1/0?(c=e!=e?1:0,l=p):(l=o(i(e)/a),e*(s=r(2,-l))<1&&(l--,s*=2),(e+=l+h>=1?v/s:v*r(2,1-h))*s>=2&&(l++,s/=2),l+h>=p?(c=0,l=p):l+h>=1?(c=(e*s-1)*r(2,t),l+=h):(c=e*r(2,h-1)*r(2,t),l=0));t>=8;f[g++]=255&c,c/=256,t-=8);for(l=l<0;f[g++]=255&l,l/=256,d-=8);return f[--g]|=128*y,f},unpack:function(e,t){var n,o=e.length,i=8*o-t-1,a=(1<>1,l=i-7,c=o-1,s=e[c--],f=127&s;for(s>>=7;l>0;f=256*f+e[c],c--,l-=8);for(n=f&(1<<-l)-1,f>>=-l,l+=t;l>0;n=256*n+e[c],c--,l-=8);if(0===f)f=1-u;else{if(f===a)return n?NaN:s?-1/0:1/0;n+=r(2,t),f-=u}return(s?-1:1)*n*r(2,f-t)}}},function(e,t,n){var r=n(23),o=n(234),i=n(9)("toPrimitive"),a=Date.prototype;i in a||r(a,i,o)},function(e,t,n){"use strict";var r=n(7),o=n(35);e.exports=function(e){if("string"!==e&&"number"!==e&&"default"!==e)throw TypeError("Incorrect hint");return o(r(this),"number"!==e)}},function(e,t,n){"use strict";var r=n(6),o=n(13),i=n(32),a=n(9)("hasInstance"),u=Function.prototype;a in u||o.f(u,a,{value:function(e){if("function"!=typeof this||!r(e))return!1;if(!r(this.prototype))return e instanceof this;for(;e=i(e);)if(this.prototype===e)return!0;return!1}})},function(e,t,n){var r=n(11),o=n(13).f,i=Function.prototype,a=i.toString,u=/^\s*function ([^ (]*)/,l="name";r&&!(l in i)&&o(i,l,{configurable:!0,get:function(){try{return a.call(this).match(u)[1]}catch(e){return""}}})},function(e,t,n){"use strict";var r=n(88),o=n(172);e.exports=r("Map",(function(e){return function(){return e(this,arguments.length?arguments[0]:void 0)}}),o)},function(e,t,n){var r=n(1),o=n(173),i=Math.acosh,a=Math.log,u=Math.sqrt,l=Math.LN2;r({target:"Math",stat:!0,forced:!i||710!=Math.floor(i(Number.MAX_VALUE))||i(1/0)!=1/0},{acosh:function(e){return(e=+e)<1?NaN:e>94906265.62425156?a(e)+l:o(e-1+u(e-1)*u(e+1))}})},function(e,t,n){var r=n(1),o=Math.asinh,i=Math.log,a=Math.sqrt;r({target:"Math",stat:!0,forced:!(o&&1/o(0)>0)},{asinh:function e(t){return isFinite(t=+t)&&0!=t?t<0?-e(-t):i(t+a(t*t+1)):t}})},function(e,t,n){var r=n(1),o=Math.atanh,i=Math.log;r({target:"Math",stat:!0,forced:!(o&&1/o(-0)<0)},{atanh:function(e){return 0==(e=+e)?e:i((1+e)/(1-e))/2}})},function(e,t,n){var r=n(1),o=n(118),i=Math.abs,a=Math.pow;r({target:"Math",stat:!0},{cbrt:function(e){return o(e=+e)*a(i(e),1/3)}})},function(e,t,n){var r=n(1),o=Math.floor,i=Math.log,a=Math.LOG2E;r({target:"Math",stat:!0},{clz32:function(e){return(e>>>=0)?31-o(i(e+.5)*a):32}})},function(e,t,n){var r=n(1),o=n(90),i=Math.cosh,a=Math.abs,u=Math.E;r({target:"Math",stat:!0,forced:!i||i(710)===1/0},{cosh:function(e){var t=o(a(e)-1)+1;return(t+1/(t*u*u))*(u/2)}})},function(e,t,n){var r=n(1),o=n(90);r({target:"Math",stat:!0,forced:o!=Math.expm1},{expm1:o})},function(e,t,n){n(1)({target:"Math",stat:!0},{fround:n(246)})},function(e,t,n){var r=n(118),o=Math.abs,i=Math.pow,a=i(2,-52),u=i(2,-23),l=i(2,127)*(2-u),c=i(2,-126);e.exports=Math.fround||function(e){var t,n,i=o(e),s=r(e);return il||n!=n?s*(1/0):s*n}},function(e,t,n){var r=n(1),o=Math.hypot,i=Math.abs,a=Math.sqrt;r({target:"Math",stat:!0,forced:!!o&&o(1/0,NaN)!==1/0},{hypot:function(e,t){for(var n,r,o=0,u=0,l=arguments.length,c=0;u0?(r=n/c)*r:n;return c===1/0?1/0:c*a(o)}})},function(e,t,n){var r=n(1),o=n(2),i=Math.imul;r({target:"Math",stat:!0,forced:o((function(){return-5!=i(4294967295,5)||2!=i.length}))},{imul:function(e,t){var n=65535,r=+e,o=+t,i=n&r,a=n&o;return 0|i*a+((n&r>>>16)*a+i*(n&o>>>16)<<16>>>0)}})},function(e,t,n){var r=n(1),o=Math.log,i=Math.LOG10E;r({target:"Math",stat:!0},{log10:function(e){return o(e)*i}})},function(e,t,n){n(1)({target:"Math",stat:!0},{log1p:n(173)})},function(e,t,n){var r=n(1),o=Math.log,i=Math.LN2;r({target:"Math",stat:!0},{log2:function(e){return o(e)/i}})},function(e,t,n){n(1)({target:"Math",stat:!0},{sign:n(118)})},function(e,t,n){var r=n(1),o=n(2),i=n(90),a=Math.abs,u=Math.exp,l=Math.E;r({target:"Math",stat:!0,forced:o((function(){return-2e-17!=Math.sinh(-2e-17)}))},{sinh:function(e){return a(e=+e)<1?(i(e)-i(-e))/2:(u(e-1)-u(-e-1))*(l/2)}})},function(e,t,n){var r=n(1),o=n(90),i=Math.exp;r({target:"Math",stat:!0},{tanh:function(e){var t=o(e=+e),n=o(-e);return t==1/0?1:n==1/0?-1:(t-n)/(i(e)+i(-e))}})},function(e,t,n){var r=n(1),o=Math.ceil,i=Math.floor;r({target:"Math",stat:!0},{trunc:function(e){return(e>0?i:o)(e)}})},function(e,t,n){"use strict";var r=n(11),o=n(3),i=n(67),a=n(20),u=n(16),l=n(43),c=n(89),s=n(35),f=n(2),d=n(44),p=n(50).f,h=n(18).f,v=n(13).f,y=n(60).trim,g="Number",m=o.Number,b=m.prototype,w=l(d(b))==g,x=function(e){var t,n,r,o,i,a,u,l,c=s(e,!1);if("string"==typeof c&&c.length>2)if(43===(t=(c=y(c)).charCodeAt(0))||45===t){if(88===(n=c.charCodeAt(2))||120===n)return NaN}else if(48===t){switch(c.charCodeAt(1)){case 66:case 98:r=2,o=49;break;case 79:case 111:r=8,o=55;break;default:return+c}for(a=(i=c.slice(2)).length,u=0;uo)return NaN;return parseInt(i,r)}return+c};if(i(g,!m(" 0o1")||!m("0b1")||m("+0x1"))){for(var E,S=function(e){var t=arguments.length<1?0:e,n=this;return n instanceof S&&(w?f((function(){b.valueOf.call(n)})):l(n)!=g)?c(new m(x(t)),n,S):x(t)},k=r?p(m):"MAX_VALUE,MIN_VALUE,NaN,NEGATIVE_INFINITY,POSITIVE_INFINITY,EPSILON,isFinite,isInteger,isNaN,isSafeInteger,MAX_SAFE_INTEGER,MIN_SAFE_INTEGER,parseFloat,parseInt,isInteger,fromString,range".split(","),O=0;k.length>O;O++)u(m,E=k[O])&&!u(S,E)&&v(S,E,h(m,E));S.prototype=b,b.constructor=S,a(o,g,S)}},function(e,t,n){n(1)({target:"Number",stat:!0},{EPSILON:Math.pow(2,-52)})},function(e,t,n){n(1)({target:"Number",stat:!0},{isFinite:n(259)})},function(e,t,n){var r=n(3).isFinite;e.exports=Number.isFinite||function(e){return"number"==typeof e&&r(e)}},function(e,t,n){n(1)({target:"Number",stat:!0},{isInteger:n(174)})},function(e,t,n){n(1)({target:"Number",stat:!0},{isNaN:function(e){return e!=e}})},function(e,t,n){var r=n(1),o=n(174),i=Math.abs;r({target:"Number",stat:!0},{isSafeInteger:function(e){return o(e)&&i(e)<=9007199254740991}})},function(e,t,n){n(1)({target:"Number",stat:!0},{MAX_SAFE_INTEGER:9007199254740991})},function(e,t,n){n(1)({target:"Number",stat:!0},{MIN_SAFE_INTEGER:-9007199254740991})},function(e,t,n){var r=n(1),o=n(266);r({target:"Number",stat:!0,forced:Number.parseFloat!=o},{parseFloat:o})},function(e,t,n){var r=n(3),o=n(60).trim,i=n(91),a=r.parseFloat,u=1/a(i+"-0")!==-1/0;e.exports=u?function(e){var t=o(String(e)),n=a(t);return 0===n&&"-"==t.charAt(0)?-0:n}:a},function(e,t,n){var r=n(1),o=n(268);r({target:"Number",stat:!0,forced:Number.parseInt!=o},{parseInt:o})},function(e,t,n){var r=n(3),o=n(60).trim,i=n(91),a=r.parseInt,u=/^[+-]?0[Xx]/,l=8!==a(i+"08")||22!==a(i+"0x16");e.exports=l?function(e,t){var n=o(String(e));return a(n,t>>>0||(u.test(n)?16:10))}:a},function(e,t,n){"use strict";var r=n(1),o=n(29),i=n(270),a=n(119),u=n(2),l=1..toFixed,c=Math.floor,s=function e(t,n,r){return 0===n?r:n%2===1?e(t,n-1,r*t):e(t*t,n/2,r)},f=function(e,t,n){for(var r=-1,o=n;++r<6;)o+=t*e[r],e[r]=o%1e7,o=c(o/1e7)},d=function(e,t){for(var n=6,r=0;--n>=0;)r+=e[n],e[n]=c(r/t),r=r%t*1e7},p=function(e){for(var t=6,n="";--t>=0;)if(""!==n||0===t||0!==e[t]){var r=String(e[t]);n=""===n?r:n+a.call("0",7-r.length)+r}return n};r({target:"Number",proto:!0,forced:l&&("0.000"!==8e-5.toFixed(3)||"1"!==.9.toFixed(0)||"1.25"!==1.255.toFixed(2)||"1000000000000000128"!==(0xde0b6b3a7640080).toFixed(0))||!u((function(){l.call({})}))},{toFixed:function(e){var t,n,r,u,l=i(this),c=o(e),h=[0,0,0,0,0,0],v="",y="0";if(c<0||c>20)throw RangeError("Incorrect fraction digits");if(l!=l)return"NaN";if(l<=-1e21||l>=1e21)return String(l);if(l<0&&(v="-",l=-l),l>1e-21)if(n=(t=function(e){for(var t=0,n=e;n>=4096;)t+=12,n/=4096;for(;n>=2;)t+=1,n/=2;return t}(l*s(2,69,1))-69)<0?l*s(2,-t,1):l/s(2,t,1),n*=4503599627370496,(t=52-t)>0){for(f(h,0,n),r=c;r>=7;)f(h,1e7,0),r-=7;for(f(h,s(10,r,1),0),r=t-1;r>=23;)d(h,1<<23),r-=23;d(h,1<0?v+((u=y.length)<=c?"0."+a.call("0",c-u)+y:y.slice(0,u-c)+"."+y.slice(u-c)):v+y}})},function(e,t,n){var r=n(43);e.exports=function(e){if("number"!=typeof e&&"Number"!=r(e))throw TypeError("Incorrect invocation");return+e}},function(e,t,n){var r=n(1),o=n(175);r({target:"Object",stat:!0,forced:Object.assign!==o},{assign:o})},function(e,t,n){"use strict";var r=n(1),o=n(11),i=n(92),a=n(14),u=n(30),l=n(13);o&&r({target:"Object",proto:!0,forced:i},{__defineGetter__:function(e,t){l.f(a(this),e,{get:u(t),enumerable:!0,configurable:!0})}})},function(e,t,n){"use strict";var r=n(1),o=n(11),i=n(92),a=n(14),u=n(30),l=n(13);o&&r({target:"Object",proto:!0,forced:i},{__defineSetter__:function(e,t){l.f(a(this),e,{set:u(t),enumerable:!0,configurable:!0})}})},function(e,t,n){var r=n(1),o=n(176).entries;r({target:"Object",stat:!0},{entries:function(e){return o(e)}})},function(e,t,n){var r=n(1),o=n(76),i=n(2),a=n(6),u=n(54).onFreeze,l=Object.freeze;r({target:"Object",stat:!0,forced:i((function(){l(1)})),sham:!o},{freeze:function(e){return l&&a(e)?l(u(e)):e}})},function(e,t,n){var r=n(1),o=n(77),i=n(52);r({target:"Object",stat:!0},{fromEntries:function(e){var t={};return o(e,(function(e,n){i(t,e,n)}),{AS_ENTRIES:!0}),t}})},function(e,t,n){var r=n(1),o=n(2),i=n(24),a=n(18).f,u=n(11),l=o((function(){a(1)}));r({target:"Object",stat:!0,forced:!u||l,sham:!u},{getOwnPropertyDescriptor:function(e,t){return a(i(e),t)}})},function(e,t,n){var r=n(1),o=n(11),i=n(108),a=n(24),u=n(18),l=n(52);r({target:"Object",stat:!0,sham:!o},{getOwnPropertyDescriptors:function(e){for(var t,n,r=a(e),o=u.f,c=i(r),s={},f=0;c.length>f;)void 0!==(n=o(r,t=c[f++]))&&l(s,t,n);return s}})},function(e,t,n){var r=n(1),o=n(2),i=n(140).f;r({target:"Object",stat:!0,forced:o((function(){return!Object.getOwnPropertyNames(1)}))},{getOwnPropertyNames:i})},function(e,t,n){var r=n(1),o=n(2),i=n(14),a=n(32),u=n(116);r({target:"Object",stat:!0,forced:o((function(){a(1)})),sham:!u},{getPrototypeOf:function(e){return a(i(e))}})},function(e,t,n){n(1)({target:"Object",stat:!0},{is:n(177)})},function(e,t,n){var r=n(1),o=n(2),i=n(6),a=Object.isExtensible;r({target:"Object",stat:!0,forced:o((function(){a(1)}))},{isExtensible:function(e){return!!i(e)&&(!a||a(e))}})},function(e,t,n){var r=n(1),o=n(2),i=n(6),a=Object.isFrozen;r({target:"Object",stat:!0,forced:o((function(){a(1)}))},{isFrozen:function(e){return!i(e)||!!a&&a(e)}})},function(e,t,n){var r=n(1),o=n(2),i=n(6),a=Object.isSealed;r({target:"Object",stat:!0,forced:o((function(){a(1)}))},{isSealed:function(e){return!i(e)||!!a&&a(e)}})},function(e,t,n){var r=n(1),o=n(14),i=n(69);r({target:"Object",stat:!0,forced:n(2)((function(){i(1)}))},{keys:function(e){return i(o(e))}})},function(e,t,n){"use strict";var r=n(1),o=n(11),i=n(92),a=n(14),u=n(35),l=n(32),c=n(18).f;o&&r({target:"Object",proto:!0,forced:i},{__lookupGetter__:function(e){var t,n=a(this),r=u(e,!0);do{if(t=c(n,r))return t.get}while(n=l(n))}})},function(e,t,n){"use strict";var r=n(1),o=n(11),i=n(92),a=n(14),u=n(35),l=n(32),c=n(18).f;o&&r({target:"Object",proto:!0,forced:i},{__lookupSetter__:function(e){var t,n=a(this),r=u(e,!0);do{if(t=c(n,r))return t.set}while(n=l(n))}})},function(e,t,n){var r=n(1),o=n(6),i=n(54).onFreeze,a=n(76),u=n(2),l=Object.preventExtensions;r({target:"Object",stat:!0,forced:u((function(){l(1)})),sham:!a},{preventExtensions:function(e){return l&&o(e)?l(i(e)):e}})},function(e,t,n){var r=n(1),o=n(6),i=n(54).onFreeze,a=n(76),u=n(2),l=Object.seal;r({target:"Object",stat:!0,forced:u((function(){l(1)})),sham:!a},{seal:function(e){return l&&o(e)?l(i(e)):e}})},function(e,t,n){var r=n(1),o=n(176).values;r({target:"Object",stat:!0},{values:function(e){return o(e)}})},function(e,t,n){"use strict";var r,o,i,a,u=n(1),l=n(36),c=n(3),s=n(31),f=n(178),d=n(20),p=n(59),h=n(37),v=n(58),y=n(6),g=n(30),m=n(47),b=n(105),w=n(77),x=n(83),E=n(55),S=n(120).set,k=n(180),O=n(181),_=n(293),T=n(182),C=n(294),P=n(25),j=n(67),N=n(9),A=n(57),R=n(72),L=N("species"),I="Promise",M=P.get,F=P.set,U=P.getterFor(I),D=f,z=c.TypeError,B=c.document,V=c.process,W=s("fetch"),H=T.f,$=H,q=!!(B&&B.createEvent&&c.dispatchEvent),Q="function"==typeof PromiseRejectionEvent,Y="unhandledrejection",G=j(I,(function(){if(!(b(D)!==String(D))){if(66===R)return!0;if(!A&&!Q)return!0}if(l&&!D.prototype.finally)return!0;if(R>=51&&/native code/.test(D))return!1;var e=D.resolve(1),t=function(e){e((function(){}),(function(){}))};return(e.constructor={})[L]=t,!(e.then((function(){}))instanceof t)})),K=G||!x((function(e){D.all(e).catch((function(){}))})),X=function(e){var t;return!(!y(e)||"function"!=typeof(t=e.then))&&t},Z=function(e,t){if(!e.notified){e.notified=!0;var n=e.reactions;k((function(){for(var r=e.value,o=1==e.state,i=0;n.length>i;){var a,u,l,c=n[i++],s=o?c.ok:c.fail,f=c.resolve,d=c.reject,p=c.domain;try{s?(o||(2===e.rejection&&ne(e),e.rejection=1),!0===s?a=r:(p&&p.enter(),a=s(r),p&&(p.exit(),l=!0)),a===c.promise?d(z("Promise-chain cycle")):(u=X(a))?u.call(a,f,d):f(a)):d(r)}catch(h){p&&!l&&p.exit(),d(h)}}e.reactions=[],e.notified=!1,t&&!e.rejection&&ee(e)}))}},J=function(e,t,n){var r,o;q?((r=B.createEvent("Event")).promise=t,r.reason=n,r.initEvent(e,!1,!0),c.dispatchEvent(r)):r={promise:t,reason:n},!Q&&(o=c["on"+e])?o(r):e===Y&&_("Unhandled promise rejection",n)},ee=function(e){S.call(c,(function(){var t,n=e.facade,r=e.value;if(te(e)&&(t=C((function(){A?V.emit("unhandledRejection",r,n):J(Y,n,r)})),e.rejection=A||te(e)?2:1,t.error))throw t.value}))},te=function(e){return 1!==e.rejection&&!e.parent},ne=function(e){S.call(c,(function(){var t=e.facade;A?V.emit("rejectionHandled",t):J("rejectionhandled",t,e.value)}))},re=function(e,t,n){return function(r){e(t,r,n)}},oe=function(e,t,n){e.done||(e.done=!0,n&&(e=n),e.value=t,e.state=2,Z(e,!0))},ie=function e(t,n,r){if(!t.done){t.done=!0,r&&(t=r);try{if(t.facade===n)throw z("Promise can't be resolved itself");var o=X(n);o?k((function(){var r={done:!1};try{o.call(n,re(e,r,t),re(oe,r,t))}catch(i){oe(r,i,t)}})):(t.value=n,t.state=1,Z(t,!1))}catch(i){oe({done:!1},i,t)}}};G&&(D=function(e){m(this,D,I),g(e),r.call(this);var t=M(this);try{e(re(ie,t),re(oe,t))}catch(n){oe(t,n)}},(r=function(e){F(this,{type:I,done:!1,notified:!1,parent:!1,reactions:[],rejection:!1,state:0,value:void 0})}).prototype=p(D.prototype,{then:function(e,t){var n=U(this),r=H(E(this,D));return r.ok="function"!=typeof e||e,r.fail="function"==typeof t&&t,r.domain=A?V.domain:void 0,n.parent=!0,n.reactions.push(r),0!=n.state&&Z(n,!1),r.promise},catch:function(e){return this.then(void 0,e)}}),o=function(){var e=new r,t=M(e);this.promise=e,this.resolve=re(ie,t),this.reject=re(oe,t)},T.f=H=function(e){return e===D||e===i?new o(e):$(e)},l||"function"!=typeof f||(a=f.prototype.then,d(f.prototype,"then",(function(e,t){var n=this;return new D((function(e,t){a.call(n,e,t)})).then(e,t)}),{unsafe:!0}),"function"==typeof W&&u({global:!0,enumerable:!0,forced:!0},{fetch:function(e){return O(D,W.apply(c,arguments))}}))),u({global:!0,wrap:!0,forced:G},{Promise:D}),h(D,I,!1,!0),v(I),i=s(I),u({target:I,stat:!0,forced:G},{reject:function(e){var t=H(this);return t.reject.call(void 0,e),t.promise}}),u({target:I,stat:!0,forced:l||G},{resolve:function(e){return O(l&&this===i?D:this,e)}}),u({target:I,stat:!0,forced:K},{all:function(e){var t=this,n=H(t),r=n.resolve,o=n.reject,i=C((function(){var n=g(t.resolve),i=[],a=0,u=1;w(e,(function(e){var l=a++,c=!1;i.push(void 0),u++,n.call(t,e).then((function(e){c||(c=!0,i[l]=e,--u||r(i))}),o)})),--u||r(i)}));return i.error&&o(i.value),n.promise},race:function(e){var t=this,n=H(t),r=n.reject,o=C((function(){var o=g(t.resolve);w(e,(function(e){o.call(t,e).then(n.resolve,r)}))}));return o.error&&r(o.value),n.promise}})},function(e,t,n){var r=n(82);e.exports=/web0s(?!.*chrome)/i.test(r)},function(e,t,n){var r=n(3);e.exports=function(e,t){var n=r.console;n&&n.error&&(1===arguments.length?n.error(e):n.error(e,t))}},function(e,t){e.exports=function(e){try{return{error:!1,value:e()}}catch(t){return{error:!0,value:t}}}},function(e,t,n){"use strict";var r=n(1),o=n(36),i=n(178),a=n(2),u=n(31),l=n(55),c=n(181),s=n(20);r({target:"Promise",proto:!0,real:!0,forced:!!i&&a((function(){i.prototype.finally.call({then:function(){}},(function(){}))}))},{finally:function(e){var t=l(this,u("Promise")),n="function"==typeof e;return this.then(n?function(n){return c(t,e()).then((function(){return n}))}:e,n?function(n){return c(t,e()).then((function(){throw n}))}:e)}}),o||"function"!=typeof i||i.prototype.finally||s(i.prototype,"finally",u("Promise").prototype.finally)},function(e,t,n){var r=n(1),o=n(31),i=n(30),a=n(7),u=n(2),l=o("Reflect","apply"),c=Function.apply;r({target:"Reflect",stat:!0,forced:!u((function(){l((function(){}))}))},{apply:function(e,t,n){return i(e),a(n),l?l(e,t,n):c.call(e,t,n)}})},function(e,t,n){var r=n(1),o=n(31),i=n(30),a=n(7),u=n(6),l=n(44),c=n(298),s=n(2),f=o("Reflect","construct"),d=s((function(){function e(){}return!(f((function(){}),[],e)instanceof e)})),p=!s((function(){f((function(){}))})),h=d||p;r({target:"Reflect",stat:!0,forced:h,sham:h},{construct:function(e,t){i(e),a(t);var n=arguments.length<3?e:i(arguments[2]);if(p&&!d)return f(e,t,n);if(e==n){switch(t.length){case 0:return new e;case 1:return new e(t[0]);case 2:return new e(t[0],t[1]);case 3:return new e(t[0],t[1],t[2]);case 4:return new e(t[0],t[1],t[2],t[3])}var r=[null];return r.push.apply(r,t),new(c.apply(e,r))}var o=n.prototype,s=l(u(o)?o:Object.prototype),h=Function.apply.call(e,s,t);return u(h)?h:s}})},function(e,t,n){"use strict";var r=n(30),o=n(6),i=[].slice,a={},u=function(e,t,n){if(!(t in a)){for(var r=[],o=0;o-1)&&(t=t.replace(/y/g,""));var u=a(x?new g(e,t):g(e,t),r?this:m,S);return E&&n&&h(u,{sticky:n}),u},k=function(e){e in S||u(S,e,{configurable:!0,get:function(){return g[e]},set:function(t){g[e]=t}})},O=l(g),_=0;O.length>_;)k(O[_++]);m.constructor=S,S.prototype=m,d(o,"RegExp",S)}v("RegExp")},function(e,t,n){var r=n(11),o=n(13),i=n(93),a=n(122).UNSUPPORTED_Y;r&&("g"!=/./g.flags||a)&&o.f(RegExp.prototype,"flags",{configurable:!0,get:i})},function(e,t,n){"use strict";var r=n(20),o=n(7),i=n(2),a=n(93),u="toString",l=RegExp.prototype,c=l.toString,s=i((function(){return"/a/b"!=c.call({source:"a",flags:"b"})})),f=c.name!=u;(s||f)&&r(RegExp.prototype,u,(function(){var e=o(this),t=String(e.source),n=e.flags;return"/"+t+"/"+String(void 0===n&&e instanceof RegExp&&!("flags"in l)?a.call(e):n)}),{unsafe:!0})},function(e,t,n){"use strict";var r=n(88),o=n(172);e.exports=r("Set",(function(e){return function(){return e(this,arguments.length?arguments[0]:void 0)}}),o)},function(e,t,n){"use strict";var r=n(1),o=n(84).codeAt;r({target:"String",proto:!0},{codePointAt:function(e){return o(this,e)}})},function(e,t,n){"use strict";var r=n(1),o=n(18).f,i=n(12),a=n(123),u=n(22),l=n(124),c=n(36),s="".endsWith,f=Math.min,d=l("endsWith");r({target:"String",proto:!0,forced:!(!c&&!d&&!!function(){var e=o(String.prototype,"endsWith");return e&&!e.writable}())&&!d},{endsWith:function(e){var t=String(u(this));a(e);var n=arguments.length>1?arguments[1]:void 0,r=i(t.length),o=void 0===n?r:f(i(n),r),l=String(e);return s?s.call(t,l,o):t.slice(o-l.length,o)===l}})},function(e,t,n){var r=n(1),o=n(51),i=String.fromCharCode,a=String.fromCodePoint;r({target:"String",stat:!0,forced:!!a&&1!=a.length},{fromCodePoint:function(e){for(var t,n=[],r=arguments.length,a=0;r>a;){if(t=+arguments[a++],o(t,1114111)!==t)throw RangeError(t+" is not a valid code point");n.push(t<65536?i(t):i(55296+((t-=65536)>>10),t%1024+56320))}return n.join("")}})},function(e,t,n){"use strict";var r=n(1),o=n(123),i=n(22);r({target:"String",proto:!0,forced:!n(124)("includes")},{includes:function(e){return!!~String(i(this)).indexOf(o(e),arguments.length>1?arguments[1]:void 0)}})},function(e,t,n){"use strict";var r=n(95),o=n(7),i=n(12),a=n(22),u=n(125),l=n(96);r("match",1,(function(e,t,n){return[function(t){var n=a(this),r=void 0==t?void 0:t[e];return void 0!==r?r.call(t,n):new RegExp(t)[e](String(n))},function(e){var r=n(t,e,this);if(r.done)return r.value;var a=o(e),c=String(this);if(!a.global)return l(a,c);var s=a.unicode;a.lastIndex=0;for(var f,d=[],p=0;null!==(f=l(a,c));){var h=String(f[0]);d[p]=h,""===h&&(a.lastIndex=u(c,i(a.lastIndex),s)),p++}return 0===p?null:d}]}))},function(e,t,n){"use strict";var r=n(1),o=n(184).end;r({target:"String",proto:!0,forced:n(185)},{padEnd:function(e){return o(this,e,arguments.length>1?arguments[1]:void 0)}})},function(e,t,n){"use strict";var r=n(1),o=n(184).start;r({target:"String",proto:!0,forced:n(185)},{padStart:function(e){return o(this,e,arguments.length>1?arguments[1]:void 0)}})},function(e,t,n){var r=n(1),o=n(24),i=n(12);r({target:"String",stat:!0},{raw:function(e){for(var t=o(e.raw),n=i(t.length),r=arguments.length,a=[],u=0;n>u;)a.push(String(t[u++])),u=O&&(k+=g.slice(O,C)+R,O=C+T.length)}return k+g.slice(O)}]}))},function(e,t,n){var r=n(14),o=Math.floor,i="".replace,a=/\$([$&'`]|\d\d?|<[^>]*>)/g,u=/\$([$&'`]|\d\d?)/g;e.exports=function(e,t,n,l,c,s){var f=n+e.length,d=l.length,p=u;return void 0!==c&&(c=r(c),p=a),i.call(s,p,(function(r,i){var a;switch(i.charAt(0)){case"$":return"$";case"&":return e;case"`":return t.slice(0,n);case"'":return t.slice(f);case"<":a=c[i.slice(1,-1)];break;default:var u=+i;if(0===u)return r;if(u>d){var s=o(u/10);return 0===s?r:s<=d?void 0===l[s-1]?i.charAt(1):l[s-1]+i.charAt(1):r}a=l[u-1]}return void 0===a?"":a}))}},function(e,t,n){"use strict";var r=n(95),o=n(7),i=n(22),a=n(177),u=n(96);r("search",1,(function(e,t,n){return[function(t){var n=i(this),r=void 0==t?void 0:t[e];return void 0!==r?r.call(t,n):new RegExp(t)[e](String(n))},function(e){var r=n(t,e,this);if(r.done)return r.value;var i=o(e),l=String(this),c=i.lastIndex;a(c,0)||(i.lastIndex=0);var s=u(i,l);return a(i.lastIndex,c)||(i.lastIndex=c),null===s?-1:s.index}]}))},function(e,t,n){"use strict";var r=n(95),o=n(121),i=n(7),a=n(22),u=n(55),l=n(125),c=n(12),s=n(96),f=n(94),d=n(2),p=[].push,h=Math.min,v=4294967295,y=!d((function(){return!RegExp(v,"y")}));r("split",2,(function(e,t,n){var r;return r="c"=="abbc".split(/(b)*/)[1]||4!="test".split(/(?:)/,-1).length||2!="ab".split(/(?:ab)*/).length||4!=".".split(/(.?)(.?)/).length||".".split(/()()/).length>1||"".split(/.?/).length?function(e,n){var r=String(a(this)),i=void 0===n?v:n>>>0;if(0===i)return[];if(void 0===e)return[r];if(!o(e))return t.call(r,e,i);for(var u,l,c,s=[],d=(e.ignoreCase?"i":"")+(e.multiline?"m":"")+(e.unicode?"u":"")+(e.sticky?"y":""),h=0,y=new RegExp(e.source,d+"g");(u=f.call(y,r))&&!((l=y.lastIndex)>h&&(s.push(r.slice(h,u.index)),u.length>1&&u.index=i));)y.lastIndex===u.index&&y.lastIndex++;return h===r.length?!c&&y.test("")||s.push(""):s.push(r.slice(h)),s.length>i?s.slice(0,i):s}:"0".split(void 0,0).length?function(e,n){return void 0===e&&0===n?[]:t.call(this,e,n)}:t,[function(t,n){var o=a(this),i=void 0==t?void 0:t[e];return void 0!==i?i.call(t,o,n):r.call(String(o),t,n)},function(e,o){var a=n(r,e,this,o,r!==t);if(a.done)return a.value;var f=i(e),d=String(this),p=u(f,RegExp),g=f.unicode,m=(f.ignoreCase?"i":"")+(f.multiline?"m":"")+(f.unicode?"u":"")+(y?"y":"g"),b=new p(y?f:"^(?:"+f.source+")",m),w=void 0===o?v:o>>>0;if(0===w)return[];if(0===d.length)return null===s(b,d)?[d]:[];for(var x=0,E=0,S=[];E1?arguments[1]:void 0,t.length)),r=String(e);return s?s.call(t,r,n):t.slice(n,n+r.length)===r}})},function(e,t,n){"use strict";var r=n(1),o=n(60).trim;r({target:"String",proto:!0,forced:n(126)("trim")},{trim:function(){return o(this)}})},function(e,t,n){"use strict";var r=n(1),o=n(60).end,i=n(126)("trimEnd"),a=i?function(){return o(this)}:"".trimEnd;r({target:"String",proto:!0,forced:i},{trimEnd:a,trimRight:a})},function(e,t,n){"use strict";var r=n(1),o=n(60).start,i=n(126)("trimStart"),a=i?function(){return o(this)}:"".trimStart;r({target:"String",proto:!0,forced:i},{trimStart:a,trimLeft:a})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("anchor")},{anchor:function(e){return o(this,"a","name",e)}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("big")},{big:function(){return o(this,"big","","")}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("blink")},{blink:function(){return o(this,"blink","","")}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("bold")},{bold:function(){return o(this,"b","","")}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("fixed")},{fixed:function(){return o(this,"tt","","")}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("fontcolor")},{fontcolor:function(e){return o(this,"font","color",e)}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("fontsize")},{fontsize:function(e){return o(this,"font","size",e)}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("italics")},{italics:function(){return o(this,"i","","")}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("link")},{link:function(e){return o(this,"a","href",e)}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("small")},{small:function(){return o(this,"small","","")}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("strike")},{strike:function(){return o(this,"strike","","")}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("sub")},{sub:function(){return o(this,"sub","","")}})},function(e,t,n){"use strict";var r=n(1),o=n(26);r({target:"String",proto:!0,forced:n(27)("sup")},{sup:function(){return o(this,"sup","","")}})},function(e,t,n){n(39)("Float32",(function(e){return function(t,n,r){return e(this,t,n,r)}}))},function(e,t,n){var r=n(29);e.exports=function(e){var t=r(e);if(t<0)throw RangeError("The argument can't be less than 0");return t}},function(e,t,n){n(39)("Float64",(function(e){return function(t,n,r){return e(this,t,n,r)}}))},function(e,t,n){n(39)("Int8",(function(e){return function(t,n,r){return e(this,t,n,r)}}))},function(e,t,n){n(39)("Int16",(function(e){return function(t,n,r){return e(this,t,n,r)}}))},function(e,t,n){n(39)("Int32",(function(e){return function(t,n,r){return e(this,t,n,r)}}))},function(e,t,n){n(39)("Uint8",(function(e){return function(t,n,r){return e(this,t,n,r)}}))},function(e,t,n){n(39)("Uint8",(function(e){return function(t,n,r){return e(this,t,n,r)}}),!0)},function(e,t,n){n(39)("Uint16",(function(e){return function(t,n,r){return e(this,t,n,r)}}))},function(e,t,n){n(39)("Uint32",(function(e){return function(t,n,r){return e(this,t,n,r)}}))},function(e,t,n){"use strict";var r=n(10),o=n(166),i=r.aTypedArray;(0,r.exportTypedArrayMethod)("copyWithin",(function(e,t){return o.call(i(this),e,t,arguments.length>2?arguments[2]:void 0)}))},function(e,t,n){"use strict";var r=n(10),o=n(21).every,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("every",(function(e){return o(i(this),e,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(10),o=n(117),i=r.aTypedArray;(0,r.exportTypedArrayMethod)("fill",(function(e){return o.apply(i(this),arguments)}))},function(e,t,n){"use strict";var r=n(10),o=n(21).filter,i=n(358),a=r.aTypedArray;(0,r.exportTypedArrayMethod)("filter",(function(e){var t=o(a(this),e,arguments.length>1?arguments[1]:void 0);return i(this,t)}))},function(e,t,n){var r=n(10).aTypedArrayConstructor,o=n(55);e.exports=function(e,t){for(var n=o(e,e.constructor),i=0,a=t.length,u=new(r(n))(a);a>i;)u[i]=t[i++];return u}},function(e,t,n){"use strict";var r=n(10),o=n(21).find,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("find",(function(e){return o(i(this),e,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(10),o=n(21).findIndex,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("findIndex",(function(e){return o(i(this),e,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(10),o=n(21).forEach,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("forEach",(function(e){o(i(this),e,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(127);(0,n(10).exportTypedArrayStaticMethod)("from",n(187),r)},function(e,t,n){"use strict";var r=n(10),o=n(66).includes,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("includes",(function(e){return o(i(this),e,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(10),o=n(66).indexOf,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("indexOf",(function(e){return o(i(this),e,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(3),o=n(10),i=n(86),a=n(9)("iterator"),u=r.Uint8Array,l=i.values,c=i.keys,s=i.entries,f=o.aTypedArray,d=o.exportTypedArrayMethod,p=u&&u.prototype[a],h=!!p&&("values"==p.name||void 0==p.name),v=function(){return l.call(f(this))};d("entries",(function(){return s.call(f(this))})),d("keys",(function(){return c.call(f(this))})),d("values",v,!h),d(a,v,!h)},function(e,t,n){"use strict";var r=n(10),o=r.aTypedArray,i=r.exportTypedArrayMethod,a=[].join;i("join",(function(e){return a.apply(o(this),arguments)}))},function(e,t,n){"use strict";var r=n(10),o=n(168),i=r.aTypedArray;(0,r.exportTypedArrayMethod)("lastIndexOf",(function(e){return o.apply(i(this),arguments)}))},function(e,t,n){"use strict";var r=n(10),o=n(21).map,i=n(55),a=r.aTypedArray,u=r.aTypedArrayConstructor;(0,r.exportTypedArrayMethod)("map",(function(e){return o(a(this),e,arguments.length>1?arguments[1]:void 0,(function(e,t){return new(u(i(e,e.constructor)))(t)}))}))},function(e,t,n){"use strict";var r=n(10),o=n(127),i=r.aTypedArrayConstructor;(0,r.exportTypedArrayStaticMethod)("of",(function(){for(var e=0,t=arguments.length,n=new(i(this))(t);t>e;)n[e]=arguments[e++];return n}),o)},function(e,t,n){"use strict";var r=n(10),o=n(87).left,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("reduce",(function(e){return o(i(this),e,arguments.length,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(10),o=n(87).right,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("reduceRight",(function(e){return o(i(this),e,arguments.length,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(10),o=r.aTypedArray,i=r.exportTypedArrayMethod,a=Math.floor;i("reverse",(function(){for(var e,t=this,n=o(t).length,r=a(n/2),i=0;i1?arguments[1]:void 0,1),n=this.length,r=a(e),u=o(r.length),c=0;if(u+t>n)throw RangeError("Wrong length");for(;ci;)s[i]=n[i++];return s}),i((function(){new Int8Array(1).slice()})))},function(e,t,n){"use strict";var r=n(10),o=n(21).some,i=r.aTypedArray;(0,r.exportTypedArrayMethod)("some",(function(e){return o(i(this),e,arguments.length>1?arguments[1]:void 0)}))},function(e,t,n){"use strict";var r=n(10),o=r.aTypedArray,i=r.exportTypedArrayMethod,a=[].sort;i("sort",(function(e){return a.call(o(this),e)}))},function(e,t,n){"use strict";var r=n(10),o=n(12),i=n(51),a=n(55),u=r.aTypedArray;(0,r.exportTypedArrayMethod)("subarray",(function(e,t){var n=u(this),r=n.length,l=i(e,r);return new(a(n,n.constructor))(n.buffer,n.byteOffset+l*n.BYTES_PER_ELEMENT,o((void 0===t?r:i(t,r))-l))}))},function(e,t,n){"use strict";var r=n(3),o=n(10),i=n(2),a=r.Int8Array,u=o.aTypedArray,l=o.exportTypedArrayMethod,c=[].toLocaleString,s=[].slice,f=!!a&&i((function(){c.call(new a(1))}));l("toLocaleString",(function(){return c.apply(f?s.call(u(this)):u(this),arguments)}),i((function(){return[1,2].toLocaleString()!=new a([1,2]).toLocaleString()}))||!i((function(){a.prototype.toLocaleString.call([1,2])})))},function(e,t,n){"use strict";var r=n(10).exportTypedArrayMethod,o=n(2),i=n(3).Uint8Array,a=i&&i.prototype||{},u=[].toString,l=[].join;o((function(){u.call({})}))&&(u=function(){return l.call(this)});var c=a.toString!=u;r("toString",u,c)},function(e,t,n){"use strict";var r,o=n(3),i=n(59),a=n(54),u=n(88),l=n(188),c=n(6),s=n(25).enforce,f=n(133),d=!o.ActiveXObject&&"ActiveXObject"in o,p=Object.isExtensible,h=function(e){return function(){return e(this,arguments.length?arguments[0]:void 0)}},v=e.exports=u("WeakMap",h,l);if(f&&d){r=l.getConstructor(h,"WeakMap",!0),a.REQUIRED=!0;var y=v.prototype,g=y.delete,m=y.has,b=y.get,w=y.set;i(y,{delete:function(e){if(c(e)&&!p(e)){var t=s(this);return t.frozen||(t.frozen=new r),g.call(this,e)||t.frozen.delete(e)}return g.call(this,e)},has:function(e){if(c(e)&&!p(e)){var t=s(this);return t.frozen||(t.frozen=new r),m.call(this,e)||t.frozen.has(e)}return m.call(this,e)},get:function(e){if(c(e)&&!p(e)){var t=s(this);return t.frozen||(t.frozen=new r),m.call(this,e)?b.call(this,e):t.frozen.get(e)}return b.call(this,e)},set:function(e,t){if(c(e)&&!p(e)){var n=s(this);n.frozen||(n.frozen=new r),m.call(this,e)?w.call(this,e,t):n.frozen.set(e,t)}else w.call(this,e,t);return this}})}},function(e,t,n){"use strict";n(88)("WeakSet",(function(e){return function(){return e(this,arguments.length?arguments[0]:void 0)}}),n(188))},function(e,t,n){var r=n(3),o=n(189),i=n(383),a=n(23);for(var u in o){var l=r[u],c=l&&l.prototype;if(c&&c.forEach!==i)try{a(c,"forEach",i)}catch(s){c.forEach=i}}},function(e,t,n){"use strict";var r=n(21).forEach,o=n(53)("forEach");e.exports=o?[].forEach:function(e){return r(this,e,arguments.length>1?arguments[1]:void 0)}},function(e,t,n){var r=n(3),o=n(189),i=n(86),a=n(23),u=n(9),l=u("iterator"),c=u("toStringTag"),s=i.values;for(var f in o){var d=r[f],p=d&&d.prototype;if(p){if(p[l]!==s)try{a(p,l,s)}catch(v){p[l]=s}if(p[c]||a(p,c,f),o[f])for(var h in i)if(p[h]!==i[h])try{a(p,h,i[h])}catch(v){p[h]=i[h]}}}},function(e,t,n){var r=n(1),o=n(3),i=n(120);r({global:!0,bind:!0,enumerable:!0,forced:!o.setImmediate||!o.clearImmediate},{setImmediate:i.set,clearImmediate:i.clear})},function(e,t,n){var r=n(1),o=n(3),i=n(180),a=n(57),u=o.process;r({global:!0,enumerable:!0,noTargetGet:!0},{queueMicrotask:function(e){var t=a&&u.domain;i(t?t.bind(e):e)}})},function(e,t,n){"use strict";n(114);var r,o=n(1),i=n(11),a=n(190),u=n(3),l=n(138),c=n(20),s=n(47),f=n(16),d=n(175),p=n(157),h=n(84).codeAt,v=n(388),y=n(37),g=n(191),m=n(25),b=u.URL,w=g.URLSearchParams,x=g.getState,E=m.set,S=m.getterFor("URL"),k=Math.floor,O=Math.pow,_="Invalid scheme",T="Invalid host",C="Invalid port",P=/[A-Za-z]/,j=/[\d+-.A-Za-z]/,N=/\d/,A=/^(0x|0X)/,R=/^[0-7]+$/,L=/^\d+$/,I=/^[\dA-Fa-f]+$/,M=/[\u0000\t\u000A\u000D #%/:?@[\\]]/,F=/[\u0000\t\u000A\u000D #/:?@[\\]]/,U=/^[\u0000-\u001F ]+|[\u0000-\u001F ]+$/g,D=/[\t\u000A\u000D]/g,z=function(e,t){var n,r,o;if("["==t.charAt(0)){if("]"!=t.charAt(t.length-1))return T;if(!(n=V(t.slice(1,-1))))return T;e.host=n}else if(K(e)){if(t=v(t),M.test(t))return T;if(null===(n=B(t)))return T;e.host=n}else{if(F.test(t))return T;for(n="",r=p(t),o=0;o4)return e;for(n=[],r=0;r1&&"0"==o.charAt(0)&&(i=A.test(o)?16:8,o=o.slice(8==i?1:2)),""===o)a=0;else{if(!(10==i?L:8==i?R:I).test(o))return e;a=parseInt(o,i)}n.push(a)}for(r=0;r=O(256,5-t))return null}else if(a>255)return null;for(u=n.pop(),r=0;r6)return;for(r=0;d();){if(o=null,r>0){if(!("."==d()&&r<4))return;f++}if(!N.test(d()))return;for(;N.test(d());){if(i=parseInt(d(),10),null===o)o=i;else{if(0==o)return;o=10*o+i}if(o>255)return;f++}l[c]=256*l[c]+o,2!=++r&&4!=r||c++}if(4!=r)return;break}if(":"==d()){if(f++,!d())return}else if(d())return;l[c++]=t}else{if(null!==s)return;f++,s=++c}}if(null!==s)for(a=c-s,c=7;0!=c&&a>0;)u=l[c],l[c--]=l[s+a-1],l[s+--a]=u;else if(8!=c)return;return l},W=function(e){var t,n,r,o;if("number"==typeof e){for(t=[],n=0;n<4;n++)t.unshift(e%256),e=k(e/256);return t.join(".")}if("object"==typeof e){for(t="",r=function(e){for(var t=null,n=1,r=null,o=0,i=0;i<8;i++)0!==e[i]?(o>n&&(t=r,n=o),r=null,o=0):(null===r&&(r=i),++o);return o>n&&(t=r,n=o),t}(e),n=0;n<8;n++)o&&0===e[n]||(o&&(o=!1),r===n?(t+=n?":":"::",o=!0):(t+=e[n].toString(16),n<7&&(t+=":")));return"["+t+"]"}return e},H={},$=d({},H,{" ":1,'"':1,"<":1,">":1,"`":1}),q=d({},$,{"#":1,"?":1,"{":1,"}":1}),Q=d({},q,{"/":1,":":1,";":1,"=":1,"@":1,"[":1,"\\":1,"]":1,"^":1,"|":1}),Y=function(e,t){var n=h(e,0);return n>32&&n<127&&!f(t,e)?e:encodeURIComponent(e)},G={ftp:21,file:null,http:80,https:443,ws:80,wss:443},K=function(e){return f(G,e.scheme)},X=function(e){return""!=e.username||""!=e.password},Z=function(e){return!e.host||e.cannotBeABaseURL||"file"==e.scheme},J=function(e,t){var n;return 2==e.length&&P.test(e.charAt(0))&&(":"==(n=e.charAt(1))||!t&&"|"==n)},ee=function(e){var t;return e.length>1&&J(e.slice(0,2))&&(2==e.length||"/"===(t=e.charAt(2))||"\\"===t||"?"===t||"#"===t)},te=function(e){var t=e.path,n=t.length;!n||"file"==e.scheme&&1==n&&J(t[0],!0)||t.pop()},ne=function(e){return"."===e||"%2e"===e.toLowerCase()},re={},oe={},ie={},ae={},ue={},le={},ce={},se={},fe={},de={},pe={},he={},ve={},ye={},ge={},me={},be={},we={},xe={},Ee={},Se={},ke=function(e,t,n,o){var i,a,u,l,c,s=n||re,d=0,h="",v=!1,y=!1,g=!1;for(n||(e.scheme="",e.username="",e.password="",e.host=null,e.port=null,e.path=[],e.query=null,e.fragment=null,e.cannotBeABaseURL=!1,t=t.replace(U,"")),t=t.replace(D,""),i=p(t);d<=i.length;){switch(a=i[d],s){case re:if(!a||!P.test(a)){if(n)return _;s=ie;continue}h+=a.toLowerCase(),s=oe;break;case oe:if(a&&(j.test(a)||"+"==a||"-"==a||"."==a))h+=a.toLowerCase();else{if(":"!=a){if(n)return _;h="",s=ie,d=0;continue}if(n&&(K(e)!=f(G,h)||"file"==h&&(X(e)||null!==e.port)||"file"==e.scheme&&!e.host))return;if(e.scheme=h,n)return void(K(e)&&G[e.scheme]==e.port&&(e.port=null));h="","file"==e.scheme?s=ye:K(e)&&o&&o.scheme==e.scheme?s=ae:K(e)?s=se:"/"==i[d+1]?(s=ue,d++):(e.cannotBeABaseURL=!0,e.path.push(""),s=xe)}break;case ie:if(!o||o.cannotBeABaseURL&&"#"!=a)return _;if(o.cannotBeABaseURL&&"#"==a){e.scheme=o.scheme,e.path=o.path.slice(),e.query=o.query,e.fragment="",e.cannotBeABaseURL=!0,s=Se;break}s="file"==o.scheme?ye:le;continue;case ae:if("/"!=a||"/"!=i[d+1]){s=le;continue}s=fe,d++;break;case ue:if("/"==a){s=de;break}s=we;continue;case le:if(e.scheme=o.scheme,a==r)e.username=o.username,e.password=o.password,e.host=o.host,e.port=o.port,e.path=o.path.slice(),e.query=o.query;else if("/"==a||"\\"==a&&K(e))s=ce;else if("?"==a)e.username=o.username,e.password=o.password,e.host=o.host,e.port=o.port,e.path=o.path.slice(),e.query="",s=Ee;else{if("#"!=a){e.username=o.username,e.password=o.password,e.host=o.host,e.port=o.port,e.path=o.path.slice(),e.path.pop(),s=we;continue}e.username=o.username,e.password=o.password,e.host=o.host,e.port=o.port,e.path=o.path.slice(),e.query=o.query,e.fragment="",s=Se}break;case ce:if(!K(e)||"/"!=a&&"\\"!=a){if("/"!=a){e.username=o.username,e.password=o.password,e.host=o.host,e.port=o.port,s=we;continue}s=de}else s=fe;break;case se:if(s=fe,"/"!=a||"/"!=h.charAt(d+1))continue;d++;break;case fe:if("/"!=a&&"\\"!=a){s=de;continue}break;case de:if("@"==a){v&&(h="%40"+h),v=!0,u=p(h);for(var m=0;m65535)return C;e.port=K(e)&&x===G[e.scheme]?null:x,h=""}if(n)return;s=be;continue}return C}h+=a;break;case ye:if(e.scheme="file","/"==a||"\\"==a)s=ge;else{if(!o||"file"!=o.scheme){s=we;continue}if(a==r)e.host=o.host,e.path=o.path.slice(),e.query=o.query;else if("?"==a)e.host=o.host,e.path=o.path.slice(),e.query="",s=Ee;else{if("#"!=a){ee(i.slice(d).join(""))||(e.host=o.host,e.path=o.path.slice(),te(e)),s=we;continue}e.host=o.host,e.path=o.path.slice(),e.query=o.query,e.fragment="",s=Se}}break;case ge:if("/"==a||"\\"==a){s=me;break}o&&"file"==o.scheme&&!ee(i.slice(d).join(""))&&(J(o.path[0],!0)?e.path.push(o.path[0]):e.host=o.host),s=we;continue;case me:if(a==r||"/"==a||"\\"==a||"?"==a||"#"==a){if(!n&&J(h))s=we;else if(""==h){if(e.host="",n)return;s=be}else{if(l=z(e,h))return l;if("localhost"==e.host&&(e.host=""),n)return;h="",s=be}continue}h+=a;break;case be:if(K(e)){if(s=we,"/"!=a&&"\\"!=a)continue}else if(n||"?"!=a)if(n||"#"!=a){if(a!=r&&(s=we,"/"!=a))continue}else e.fragment="",s=Se;else e.query="",s=Ee;break;case we:if(a==r||"/"==a||"\\"==a&&K(e)||!n&&("?"==a||"#"==a)){if(".."===(c=(c=h).toLowerCase())||"%2e."===c||".%2e"===c||"%2e%2e"===c?(te(e),"/"==a||"\\"==a&&K(e)||e.path.push("")):ne(h)?"/"==a||"\\"==a&&K(e)||e.path.push(""):("file"==e.scheme&&!e.path.length&&J(h)&&(e.host&&(e.host=""),h=h.charAt(0)+":"),e.path.push(h)),h="","file"==e.scheme&&(a==r||"?"==a||"#"==a))for(;e.path.length>1&&""===e.path[0];)e.path.shift();"?"==a?(e.query="",s=Ee):"#"==a&&(e.fragment="",s=Se)}else h+=Y(a,q);break;case xe:"?"==a?(e.query="",s=Ee):"#"==a?(e.fragment="",s=Se):a!=r&&(e.path[0]+=Y(a,H));break;case Ee:n||"#"!=a?a!=r&&("'"==a&&K(e)?e.query+="%27":e.query+="#"==a?"%23":Y(a,H)):(e.fragment="",s=Se);break;case Se:a!=r&&(e.fragment+=Y(a,$))}d++}},Oe=function(e){var t,n,r=s(this,Oe,"URL"),o=arguments.length>1?arguments[1]:void 0,a=String(e),u=E(r,{type:"URL"});if(void 0!==o)if(o instanceof Oe)t=S(o);else if(n=ke(t={},String(o)))throw TypeError(n);if(n=ke(u,a,null,t))throw TypeError(n);var l=u.searchParams=new w,c=x(l);c.updateSearchParams(u.query),c.updateURL=function(){u.query=String(l)||null},i||(r.href=Te.call(r),r.origin=Ce.call(r),r.protocol=Pe.call(r),r.username=je.call(r),r.password=Ne.call(r),r.host=Ae.call(r),r.hostname=Re.call(r),r.port=Le.call(r),r.pathname=Ie.call(r),r.search=Me.call(r),r.searchParams=Fe.call(r),r.hash=Ue.call(r))},_e=Oe.prototype,Te=function(){var e=S(this),t=e.scheme,n=e.username,r=e.password,o=e.host,i=e.port,a=e.path,u=e.query,l=e.fragment,c=t+":";return null!==o?(c+="//",X(e)&&(c+=n+(r?":"+r:"")+"@"),c+=W(o),null!==i&&(c+=":"+i)):"file"==t&&(c+="//"),c+=e.cannotBeABaseURL?a[0]:a.length?"/"+a.join("/"):"",null!==u&&(c+="?"+u),null!==l&&(c+="#"+l),c},Ce=function(){var e=S(this),t=e.scheme,n=e.port;if("blob"==t)try{return new URL(t.path[0]).origin}catch(r){return"null"}return"file"!=t&&K(e)?t+"://"+W(e.host)+(null!==n?":"+n:""):"null"},Pe=function(){return S(this).scheme+":"},je=function(){return S(this).username},Ne=function(){return S(this).password},Ae=function(){var e=S(this),t=e.host,n=e.port;return null===t?"":null===n?W(t):W(t)+":"+n},Re=function(){var e=S(this).host;return null===e?"":W(e)},Le=function(){var e=S(this).port;return null===e?"":String(e)},Ie=function(){var e=S(this),t=e.path;return e.cannotBeABaseURL?t[0]:t.length?"/"+t.join("/"):""},Me=function(){var e=S(this).query;return e?"?"+e:""},Fe=function(){return S(this).searchParams},Ue=function(){var e=S(this).fragment;return e?"#"+e:""},De=function(e,t){return{get:e,set:t,configurable:!0,enumerable:!0}};if(i&&l(_e,{href:De(Te,(function(e){var t=S(this),n=String(e),r=ke(t,n);if(r)throw TypeError(r);x(t.searchParams).updateSearchParams(t.query)})),origin:De(Ce),protocol:De(Pe,(function(e){var t=S(this);ke(t,String(e)+":",re)})),username:De(je,(function(e){var t=S(this),n=p(String(e));if(!Z(t)){t.username="";for(var r=0;r>1,e+=u(e/t);e>455;r+=36)e=u(e/35);return u(r+36*e/(e+38))},f=function(e){var t,n,o=[],i=(e=function(e){for(var t=[],n=0,r=e.length;n=55296&&o<=56319&&n=f&&nu((r-d)/g))throw RangeError(a);for(d+=(y-f)*g,f=y,t=0;tr)throw RangeError(a);if(n==f){for(var m=d,b=36;;b+=36){var w=b<=p?1:b>=p+26?26:b-p;if(m