Skip to content

Commit

Permalink
ammend add_schema_metadata to only accept required params. add docstr…
Browse files Browse the repository at this point in the history
…ing. move test_lake from pycrunch tests to functests in zoom
  • Loading branch information
Alig1493 committed Apr 25, 2024
1 parent e550231 commit 8d9e924
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 26 deletions.
25 changes: 19 additions & 6 deletions src/pycrunch/importing.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,21 +53,34 @@ def wait_for_batch_status(self, batch, status):
raise ValueError("The batch did not reach the '%s' state in the "
"given time. Please check again later." % status)

def add_schema_metadata(self, ds, source_url, filename, fp, mimetype, schema, metadata):
response = ds.session.post(
source_url,
def add_schema_metadata(self, site, schema, metadata, filename, fp, mimetype="application/x-parquet"):
"""
Create a new Source from a parquet file using schema and metadata.
Parameters:
site (shoji.Catalog): a shoji Catalog object, from which we acquire session and sources url
schema (str): json string containing schema
metadata (str): json string containing metadata
filename (str): name of file being uploaded
fp (BufferedReader): opened file object
mimetype (str): mimetype of file being uploaded
Returns:
shoji.Entity: Shoji entity containing the payload, status_code and source_url of uploaded file
"""
response = site.session.post(
site.catalogs.sources,
files={
"uploaded_file": (filename, fp, mimetype)
},
data={
"schema": schema,
"metadata": metadata,
"crunchlake": "create",
"dataset_id": ds.get("self", "").split("/")[-2]
"dataset_id": "None"
}
)

return shoji.Entity(ds.session, body={
return shoji.Entity(site.session, body={
"status_code": response.status_code,
"payload": response.payload,
"source_url": response.headers.get("Location")
Expand Down
20 changes: 0 additions & 20 deletions tests/test_lake.py

This file was deleted.

0 comments on commit 8d9e924

Please sign in to comment.