From 9ef07e0973b468addf6aae9cd39638b23652d99f Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Thu, 16 Jan 2025 15:27:40 +0100
Subject: [PATCH 1/9] Factorize code
---
dinamis_sdk/__init__.py | 2 +-
dinamis_sdk/cli.py | 8 ++++----
dinamis_sdk/signing.py | 1 -
3 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/dinamis_sdk/__init__.py b/dinamis_sdk/__init__.py
index b117818..74ef432 100644
--- a/dinamis_sdk/__init__.py
+++ b/dinamis_sdk/__init__.py
@@ -2,7 +2,7 @@
# flake8: noqa
-__version__ = "0.4.0"
+__version__ = "0.4.1"
from dinamis_sdk.signing import (
sign,
sign_inplace,
diff --git a/dinamis_sdk/cli.py b/dinamis_sdk/cli.py
index 2d013d4..57469f7 100644
--- a/dinamis_sdk/cli.py
+++ b/dinamis_sdk/cli.py
@@ -17,7 +17,7 @@ def app() -> None:
"""Click group for dinamis sdk subcommands."""
-def http(route: str):
+def _http(route: str):
"""Perform an HTTP request."""
session = create_session()
ret = session.get(
@@ -31,17 +31,17 @@ def http(route: str):
def create_key() -> Dict[str, str]:
"""Create an API key."""
- return http("create_api_key").json()
+ return _http("create_api_key").json()
def list_keys() -> List[str]:
"""List all generated API keys."""
- return http("list_api_keys").json()
+ return _http("list_api_keys").json()
def revoke_key(key: str):
"""Revoke an API key."""
- http(f"revoke_api_key?access_key={key}")
+ _http(f"revoke_api_key?access_key={key}")
log.info(f"API key {key} revoked")
diff --git a/dinamis_sdk/signing.py b/dinamis_sdk/signing.py
index 152d57a..a203583 100644
--- a/dinamis_sdk/signing.py
+++ b/dinamis_sdk/signing.py
@@ -3,7 +3,6 @@ S3 Module.
Revamp of Microsoft Planetary Computer SAS, using S3 and custom URL signing
endpoint instead.
-
"""
import collections.abc
--
GitLab
From 9ccd38caa706b5f5f2446a75c65ad75bd802aa6e Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Thu, 16 Jan 2025 15:36:20 +0100
Subject: [PATCH 2/9] Fix put cache
---
dinamis_sdk/signing.py | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/dinamis_sdk/signing.py b/dinamis_sdk/signing.py
index a203583..6f3a559 100644
--- a/dinamis_sdk/signing.py
+++ b/dinamis_sdk/signing.py
@@ -12,7 +12,7 @@ import time
from copy import deepcopy
from datetime import datetime, timezone
from functools import singledispatch
-from typing import Any, Dict, List, Mapping, TypeVar, cast
+from typing import Any, Dict, List, Mapping, TypeVar, cast, Enum
from urllib.parse import parse_qs, urlparse
import packaging.version
@@ -152,7 +152,14 @@ def sign_string(url: str, copy: bool = True) -> str:
return sign_urls(urls=[url])[url]
-def _generic_sign_urls(urls: List[str], route: str) -> Dict[str, str]:
+class SignURLRoute(Enum):
+ """Different routes used for sign_urls."""
+
+ SIGN_URLS_GET = "sign_urls"
+ SIGN_URLS_PUT = "sign_urls_put"
+
+
+def _generic_sign_urls(urls: List[str], route: SignURLRoute) -> Dict[str, str]:
"""Sign URLs with a S3 Token.
Signing URL allows read access to files in storage.
@@ -203,12 +210,12 @@ def _generic_sign_urls(urls: List[str], route: str) -> Dict[str, str]:
def sign_urls(urls: List[str]) -> Dict[str, str]:
"""Sign multiple URLs for GET."""
- return _generic_sign_urls(urls=urls, route="sign_urls")
+ return _generic_sign_urls(urls=urls, route=SignURLRoute(SignURLRoute.SIGN_URLS_GET))
def sign_urls_put(urls: List[str]) -> Dict[str, str]:
"""Sign multiple URLs for PUT."""
- return _generic_sign_urls(urls=urls, route="sign_urls_put")
+ return _generic_sign_urls(urls=urls, route=SignURLRoute(SignURLRoute.SIGN_URLS_PUT))
def sign_url_put(url: str) -> str:
@@ -435,7 +442,7 @@ sign_reference_file = sign_mapping
def _generic_get_signed_urls(
urls: List[str],
- route: str,
+ route: SignURLRoute,
) -> Dict[str, SignedURL]:
"""
Get multiple signed URLs.
@@ -459,7 +466,7 @@ def _generic_get_signed_urls(
signed_urls = {}
for url in urls:
signed_url_in_cache = CACHE.get(url)
- if signed_url_in_cache:
+ if signed_url_in_cache and route == SignURLRoute.SIGN_URLS_GET:
log.debug("URL %s already in cache", url)
ttl = signed_url_in_cache.ttl()
log.debug("URL %s TTL is %s", url, ttl)
--
GitLab
From f0bf25f3483bd777b27dc638ddd2f84c40b7ac82 Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Thu, 16 Jan 2025 15:40:02 +0100
Subject: [PATCH 3/9] Fix typo
---
dinamis_sdk/signing.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/dinamis_sdk/signing.py b/dinamis_sdk/signing.py
index 6f3a559..48921cc 100644
--- a/dinamis_sdk/signing.py
+++ b/dinamis_sdk/signing.py
@@ -12,7 +12,8 @@ import time
from copy import deepcopy
from datetime import datetime, timezone
from functools import singledispatch
-from typing import Any, Dict, List, Mapping, TypeVar, cast, Enum
+from typing import Any, Dict, List, Mapping, TypeVar, cast
+from enum import Enum
from urllib.parse import parse_qs, urlparse
import packaging.version
--
GitLab
From 67a148feb9e2e5c69798dbd683a9bdf4fc71c95e Mon Sep 17 00:00:00 2001
From: Cresson Remi <remi.cresson@irstea.fr>
Date: Thu, 16 Jan 2025 16:07:11 +0100
Subject: [PATCH 4/9] Update file test_push.py
---
tests/test_push.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/tests/test_push.py b/tests/test_push.py
index d009e30..9c97f5d 100755
--- a/tests/test_push.py
+++ b/tests/test_push.py
@@ -7,8 +7,11 @@ local_filename = "/tmp/toto.txt"
with open(local_filename, "w") as f:
f.write("hello world")
-pushed = dinamis_sdk.push(
- local_filename=local_filename,
- target_url="https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt",
-)
+for i in range(2):
+ pushed = dinamis_sdk.push(
+ local_filename=local_filename,
+ target_url="https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt",
+ )
+ print(f"Push #{i} ok")
+
print("Done")
--
GitLab
From 58f5029955ee5047b305fd5603169ed96cef3a1a Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Thu, 16 Jan 2025 16:09:36 +0100
Subject: [PATCH 5/9] Fix
---
dinamis_sdk/signing.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dinamis_sdk/signing.py b/dinamis_sdk/signing.py
index 48921cc..4e36c11 100644
--- a/dinamis_sdk/signing.py
+++ b/dinamis_sdk/signing.py
@@ -498,7 +498,7 @@ def _generic_get_signed_urls(
params: Dict[str, Any] = {"urls": not_signed_urls_chunk}
if ENV.dinamis_sdk_url_duration:
params["duration_seconds"] = ENV.dinamis_sdk_url_duration
- response = session.post(route=route, params=params)
+ response = session.post(route=route.value, params=params)
signed_url_batch = SignedURLBatch(**response.json())
if not signed_url_batch:
raise ValueError(
--
GitLab
From 69324ddee5137f126bd62aa0ea4d5df655de26c1 Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Thu, 16 Jan 2025 16:14:15 +0100
Subject: [PATCH 6/9] Set static analysis test allow_failure to false
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b28b07a..e1bcbe9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -41,7 +41,7 @@ pip_install:
.static_analysis_base:
stage: Static Analysis
- allow_failure: true
+ allow_failure: false
flake8:
needs:
--
GitLab
From 974346035e9e30b76e30abf6d62f1e76831dd3d8 Mon Sep 17 00:00:00 2001
From: Cresson Remi <remi.cresson@irstea.fr>
Date: Thu, 16 Jan 2025 16:18:17 +0100
Subject: [PATCH 7/9] Update file test_push.py
---
tests/test_push.py | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/tests/test_push.py b/tests/test_push.py
index 9c97f5d..4aa2940 100755
--- a/tests/test_push.py
+++ b/tests/test_push.py
@@ -1,17 +1,24 @@
#!/usr/bin/env python
import dinamis_sdk
+import requests
local_filename = "/tmp/toto.txt"
with open(local_filename, "w") as f:
f.write("hello world")
-for i in range(2):
- pushed = dinamis_sdk.push(
- local_filename=local_filename,
- target_url="https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt",
- )
- print(f"Push #{i} ok")
+target_url = "https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt"
+
+dinamis_sdk.push(local_filename=local_filename, target_url=target_url)
+print("push OK")
+
+signed_url = dinamis_sdk.sign(target_url)
+print("sign OK")
+
+res = requests.get(signed_url, stream=True, timeout=10)
+assert res.status_code == 200, "Get NOK"
+print("get OK")
+
print("Done")
--
GitLab
From c64ef93633974de865aaf8e2806d8c19efeb0304 Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Thu, 16 Jan 2025 16:27:48 +0100
Subject: [PATCH 8/9] Update push test and push cache logic
---
dinamis_sdk/signing.py | 2 +-
tests/test_push.py | 13 +++++++------
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/dinamis_sdk/signing.py b/dinamis_sdk/signing.py
index 4e36c11..9621314 100644
--- a/dinamis_sdk/signing.py
+++ b/dinamis_sdk/signing.py
@@ -512,7 +512,7 @@ def _generic_get_signed_urls(
)
for url, href in signed_url_batch.hrefs.items():
signed_url = SignedURL(expiry=signed_url_batch.expiry, href=href)
- if route == "sign_urls":
+ if route == SignURLRoute.SIGN_URLS_GET:
# Only put GET urls in cache
CACHE[url] = signed_url
signed_urls[url] = signed_url
diff --git a/tests/test_push.py b/tests/test_push.py
index 4aa2940..f61c719 100755
--- a/tests/test_push.py
+++ b/tests/test_push.py
@@ -1,19 +1,20 @@
#!/usr/bin/env python
-import dinamis_sdk
import requests
-local_filename = "/tmp/toto.txt"
+import dinamis_sdk
+
+LOCAL_FILENAME = "/tmp/toto.txt"
-with open(local_filename, "w") as f:
+with open(LOCAL_FILENAME, "w") as f:
f.write("hello world")
-target_url = "https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt"
+TARGET_URL = "https://s3-data.meso.umontpellier.fr/sm1-gdc-tests/titi.txt"
-dinamis_sdk.push(local_filename=local_filename, target_url=target_url)
+dinamis_sdk.push(local_filename=LOCAL_FILENAME, target_url=TARGET_URL)
print("push OK")
-signed_url = dinamis_sdk.sign(target_url)
+signed_url = dinamis_sdk.sign(TARGET_URL)
print("sign OK")
res = requests.get(signed_url, stream=True, timeout=10)
--
GitLab
From e2f238adee97808df5f06fae9367434d6d9a6326 Mon Sep 17 00:00:00 2001
From: DE BOISSIEU FLORIAN <florian.deboissieu@inrae.fr>
Date: Thu, 16 Jan 2025 16:41:37 +0100
Subject: [PATCH 9/9] add ref to rstactheia
---
README.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/README.md b/README.md
index d80f17b..c1842f6 100644
--- a/README.md
+++ b/README.md
@@ -27,3 +27,5 @@ api = pystac_client.Client.open(
For more information read the
[documentation](https://cdos-pub.pages.mia.inra.fr/dinamis-sdk).
+
+For `R` users, check [rstactheia](https://cdos-pub.pages.mia.inra.fr/rstactheia).
--
GitLab