-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3 from lsst/tickets/OPSIM-1091
Minimum setup for unit tests to run and pass
- Loading branch information
Showing
15 changed files
with
399 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
setuptools_scm | ||
setuptools_scm_git_archive | ||
numpy | ||
matplotlib | ||
healpy | ||
pandas | ||
numexpr | ||
palpy | ||
scipy | ||
sqlalchemy | ||
astropy | ||
pytables | ||
h5py | ||
openorb | ||
openorb-data-de405 | ||
astroplan | ||
colorcet | ||
cycler | ||
george | ||
scikit-learn | ||
requests | ||
tqdm |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
from .version import __version__ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
from .data_sets import * # noqa: F403 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
__all__ = ("get_data_dir", "data_versions", "get_baseline") | ||
|
||
import glob | ||
import os | ||
|
||
|
||
def get_data_dir(): | ||
"""Get the location of the rubin_sim data directory. | ||
Returns | ||
------- | ||
data_dir : `str` | ||
Path to the rubin_sim data directory. | ||
""" | ||
# See if there is an environment variable with the path | ||
data_dir = os.getenv("RUBIN_SIM_DATA_DIR") | ||
|
||
# Set the root data directory | ||
if data_dir is None: | ||
data_dir = os.path.join(os.getenv("HOME"), "rubin_sim_data") | ||
return data_dir | ||
|
||
|
||
def get_baseline(): | ||
"""Get the path to the baseline cadence simulation sqlite file. | ||
Returns | ||
------- | ||
file : `str` | ||
Path to the baseline cadence simulation sqlite file. | ||
""" | ||
dd = get_data_dir() | ||
path = os.path.join(dd, "sim_baseline") | ||
file = glob.glob(path + "/*10yrs.db")[0] | ||
return file | ||
|
||
|
||
def data_versions(): | ||
"""Get the dictionary of source filenames in the rubin_sim data directory. | ||
Returns | ||
------- | ||
result : `dict` | ||
Data directory filenames dictionary with keys: | ||
``"name"`` | ||
Data bucket name (`str`). | ||
``"version"`` | ||
Versioned file name (`str`). | ||
""" | ||
data_dir = get_data_dir() | ||
result = None | ||
version_file = os.path.join(data_dir, "versions.txt") | ||
if os.path.isfile(version_file): | ||
with open(version_file) as f: | ||
content = f.readlines() | ||
content = [x.strip() for x in content] | ||
result = {} | ||
for line in content: | ||
ack = line.split(",") | ||
result[ack[0]] = ack[1] | ||
|
||
return result |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
__all__ = ("MyHTMLParser", "rs_download_sky") | ||
|
||
import argparse | ||
import os | ||
from html.parser import HTMLParser | ||
|
||
import requests | ||
|
||
from . import get_data_dir | ||
|
||
|
||
# Hack it up to find the filenames ending with .h5 | ||
class MyHTMLParser(HTMLParser): | ||
"""HTML parser class that uses the HTMLParser to parse a starttag. | ||
See Also | ||
-------- | ||
html.parser.HTMLParser | ||
Examples | ||
-------- | ||
To instantiate a MyHTMLParser instance: | ||
parser = MyHTMLParser() | ||
parser.handle_starttag(tag, attrs) | ||
""" | ||
|
||
def handle_starttag(self, tag, attrs): | ||
""" | ||
Handle the start tag of an element (e.g. <div id="main">). | ||
Parameters | ||
---------- | ||
tag : `str` | ||
The name of the tag converted to lower case. | ||
attrs : `list` | ||
A list of (name, value) pairs containing the attributes | ||
found inside the tag’s <> brackets | ||
""" | ||
try: | ||
self.filenames | ||
except AttributeError: | ||
setattr(self, "filenames", []) | ||
if tag == "a": | ||
if attrs[0][0] == "href": | ||
if attrs[0][1].endswith(".h5"): | ||
self.filenames.append(attrs[0][1]) | ||
|
||
|
||
def rs_download_sky(): | ||
"""Download sky files.""" | ||
parser = argparse.ArgumentParser( | ||
description="Download precomputed skybrightness files for rubin_sim package" | ||
) | ||
parser.add_argument( | ||
"-f", | ||
"--force", | ||
dest="force", | ||
default=False, | ||
action="store_true", | ||
help="Force re-download of sky brightness data.", | ||
) | ||
parser.add_argument( | ||
"--url_base", | ||
type=str, | ||
default="https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/sims_skybrightness_pre/h5_2023_09_12/", | ||
help="Root URL of download location", | ||
) | ||
args = parser.parse_args() | ||
|
||
data_dir = get_data_dir() | ||
destination = os.path.join(data_dir, "skybrightness_pre") | ||
if not os.path.isdir(data_dir): | ||
os.mkdir(data_dir) | ||
if not os.path.isdir(destination): | ||
os.mkdir(destination) | ||
|
||
# Get the index file | ||
r = requests.get(args.url_base) | ||
# Find the filenames | ||
parser = MyHTMLParser() | ||
parser.feed(r.text) | ||
parser.close() | ||
# Copy the sky data files, if they're not already present | ||
for file in parser.filenames: | ||
if not os.path.isfile(os.path.join(destination, file)) or args.force: | ||
url = args.url_base + file | ||
print(f"Downloading file {file} from {url}") | ||
r = requests.get(url) | ||
with open(os.path.join(destination, file), "wb") as f: | ||
f.write(r.content) |
Oops, something went wrong.