test script for uploads

clowder2.0/upload_multiple_files.py

@@ -0,0 +1,28 @@
+import requests
+import os
+
+url = 'http://127.0.0.1:8000/'
+username = 'ra'
+password = 'rarara'
+
+def get_token():
+    login_url = url + 'api/v2/login'
+    data = dict()
+    data = {'name':username, 'password':password}
+    r = requests.post(login_url, json=data)
+    r_json = r.json()
+    token = r_json['token']
+    return token
+
+def get_datasets():
+    token = get_token()
+    headers = {'Authorization': 'Bearer ' + token }
+    datasets_url = url + 'api/v2/datasets'
+    datasets = requests.get(datasets_url, headers=headers)
+    print('done')
+
+
+if __name__ == "__main__":
+    ds = get_datasets()
+    t = get_token()
+    print('done')

conda_related.py

@@ -0,0 +1,12 @@
+import os
+import sys
+import subprocess
+
+def get_conda_environments():
+    conda_environments = subprocess.run(['conda', 'env', 'list'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    return str(conda_environments.stdout)
+
+if __name__ == '__main__':
+    environments = get_conda_environments()
+
+    print('done')

fit-oscm-test-upload/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.9.10-slim-buster AS builder
+
+RUN python -m pip install --upgrade pip
+
+#Install Cron
+RUN apt-get update
+RUN apt-get -y install cron
+RUN apt-get -y install vim
+
+COPY test.py ./home/test.py
+
+COPY run.sh ./home/run.sh
+
+RUN  chmod +x /home/run.sh
+
+# Add crontab file in the cron directory
+ADD test-cron /etc/cron.d/test-cron
+
+# Give execution rights on the cron job
+RUN chmod 0644 /etc/cron.d/test-cron
+
+# Create the log file to be able to run tail
+RUN touch /var/log/cron.log
+
+# Run the command on container startup
+CMD cron && tail -f /var/log/cron.log
+

fit-oscm-test-upload/run.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+python test.py

fit-oscm-test-upload/test-cron

@@ -0,0 +1 @@
+*/2 * * * * * root /home/run.sh

fit-oscm-test-upload/test.py

@@ -0,0 +1,11 @@
+import datetime
+
+
+def main():
+    now = datetime.datetime.now()
+    print('running script at this time')
+    print(str(now))
+
+
+if __name__ == '__main__':
+    main()

fit-oscm-test-upload/upload_data.py

@@ -0,0 +1,67 @@
+import pyclowder
+import sys
+import pyclowder.files
+import pyclowder.datasets
+import time
+import datetime
+import os
+
+import environment
+
+
+def main():
+
+    print('environment variables')
+
+    for k, v in os.environ.items():
+        print(f'{k}={v}')
+
+    CLOWDER_URL = os.environ.get('clowder_url')
+    CLOWDER_KEY = os.environ.get('api_key')
+    SPACE_ID = os.environ.get('space_id')
+
+    print('current time')
+    collection_name = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+
+    print(collection_name)
+
+    client = pyclowder.datasets.ClowderClient(host=CLOWDER_URL, key=CLOWDER_KEY)
+
+    # creating collection in a space
+    data = dict()
+    data["name"] = collection_name
+    data["description"] = ''
+    data["space"] = SPACE_ID
+    collection_result = client.post("/collections", content=data, params=data)
+    collection_id = collection_result['id']
+
+    # creating a dataset in the space and collection
+    data = dict()
+    dataset_name = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
+
+    data["name"] = dataset_name
+    data["description"] = ''
+    data["space"] = [SPACE_ID]
+    if collection_id:
+        data["collection"] = [collection_id]
+    result = client.post("/datasets/createempty", content=data, params=data)
+    data["name"] = "brand new dataset"
+    data["description"] = "this is new"
+    data["collection"] = collection_id
+
+    result = client.post("/datasets/createempty", content=data, params=data)
+
+    # TODO upload files
+
+
+
+
+
+if __name__ == '__main__':
+
+    cycles = 0
+
+    while cycles < 5:
+        print('sleeping')
+        time.sleep(60*2)
+        main()