Skip to content

Commit

Permalink
Merge branch 'harmony-1495' of https://github.com/nasa/harmony into h…
Browse files Browse the repository at this point in the history
…armony-1495
  • Loading branch information
vinnyinverso committed Jul 19, 2023
2 parents f6fb54e + 00d7b00 commit 61fc0a6
Show file tree
Hide file tree
Showing 30 changed files with 653 additions and 1,335 deletions.
21 changes: 8 additions & 13 deletions .nsprc
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,22 @@
},
"1092429": {
"active": true,
"notes": "Ignored since we don't expose the vulnerability and there is no fix",
"expiry": "2023-10-12"
},
"1092459": {
"active": true,
"notes": "Ignored since only affects semver used for development and there is no fix",
"expiry": "2023-10-12"
"notes": "Ignored since there is no fix",
"expiry": "2023-10-01"
},
"1092460": {
"active": true,
"notes": "Ignored since only affects semver used for development and there is no fix",
"expiry": "2023-10-12"
"notes": "Ignored since there is no fix",
"expiry": "2023-10-01"
},
"1092461": {
"active": true,
"notes": "Ignored since only affects semver used for development and there is no fix",
"expiry": "2023-10-12"
"notes": "Ignored since there is no fix",
"expiry": "2023-10-01"
},
"1092470": {
"active": true,
"notes": "Ignored since we don't expose the vulnerability and there is no fix",
"expiry": "2023-10-12"
"notes": "Ignored since there is no fix",
"expiry": "2023-10-01"
}
}
9 changes: 0 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,6 @@ git clone https://github.com/nasa/harmony.git
cd harmony && ./bin/bootstrap-harmony
```

Linux Only (Handled automatically by Docker Desktop)

5. Expose the kubernetes services to the local host. These commands will block so they must be run in separate terminals.
```bash
kubectl port-forward service/harmony 3000:3000 -n harmony
```

End of Linux Only

Harmony should now be running in your Kubernetes cluster as the `harmony` service in the `harmony` namespace.

**NOTE** It may take a while for all the pods to start if this is the first time you have started
Expand Down
1 change: 0 additions & 1 deletion app/frontends/jobs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,6 @@ export async function getJobStatus(
const pagingLinks = getPagingLinks(req, pagination).map((link) => new JobLink(link));
job.links = job.links.concat(pagingLinks);
const jobForDisplay = getJobForDisplay(job, urlRoot, linkType, errors);
jobForDisplay.shareable = isJobShareable;
res.send(jobForDisplay);
} catch (e) {
req.context.logger.error(e);
Expand Down
15 changes: 13 additions & 2 deletions app/frontends/workflow-ui.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ import { Logger } from 'winston';
import { serviceNames } from '../models/services';
import { getEdlGroupInformation, isAdminUser } from '../util/edl-api';

// Default to retrieving this number of work items per page
const defaultWorkItemPageSize = 100;

/**
* Maps job status to display class.
*/
Expand Down Expand Up @@ -247,10 +250,14 @@ export async function getJobs(
const lastPage = pageLinks.find((l) => l.rel === 'last');
const nextPage = pageLinks.find((l) => l.rel === 'next');
const previousPage = pageLinks.find((l) => l.rel === 'prev');
const paginationInfo = { from: (pagination.from + 1).toLocaleString(),
to: pagination.to.toLocaleString(), total: pagination.total.toLocaleString(),
currentPage: pagination.currentPage.toLocaleString(), lastPage: pagination.lastPage.toLocaleString() };
res.render('workflow-ui/jobs/index', {
version,
page,
limit,
paginationInfo, // formatted for display
currentUser: req.user,
isAdminRoute,
jobs,
Expand Down Expand Up @@ -295,7 +302,7 @@ export async function getJob(
try {
const isAdmin = await isAdminUser(req);
const job = await getJobIfAllowed(jobID, req.user, isAdmin, req.accessToken, true);
const { page, limit } = getPagingParams(req, 1000);
const { page, limit } = getPagingParams(req, defaultWorkItemPageSize, true);
const requestQuery = keysToLowerCase(req.query);
const fromDateTime = requestQuery.fromdatetime;
const toDateTime = requestQuery.todatetime;
Expand Down Expand Up @@ -452,7 +459,7 @@ export async function getWorkItemsTable(
res.status(204).json({ status: job.status });
return;
}
const { page, limit } = getPagingParams(req, env.defaultJobListPageSize);
const { page, limit } = getPagingParams(req, defaultWorkItemPageSize, true);
const requestQuery = keysToLowerCase(req.query);
const { tableQuery } = parseQuery(requestQuery, WorkItemStatus);
const itemQuery = tableQueryToWorkItemQuery(tableQuery, jobID);
Expand All @@ -462,10 +469,14 @@ export async function getWorkItemsTable(
const lastPage = pageLinks.find((l) => l.rel === 'last');
const nextPage = pageLinks.find((l) => l.rel === 'next');
const previousPage = pageLinks.find((l) => l.rel === 'prev');
const paginationInfo = { from: (pagination.from + 1).toLocaleString(),
to: pagination.to.toLocaleString(), total: pagination.total.toLocaleString(),
currentPage: pagination.currentPage.toLocaleString(), lastPage: pagination.lastPage.toLocaleString() };
setPagingHeaders(res, pagination);
res.render('workflow-ui/job/work-items-table', {
isAdminOrLogViewer,
canShowRetryColumn: job.belongsToOrIsAdmin(req.user, isAdmin),
paginationInfo,
job,
statusClass: statusClass[job.status],
workItems,
Expand Down
2 changes: 0 additions & 2 deletions app/models/job.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,6 @@ export class JobForDisplay {

errors?: JobError[];

shareable?: boolean;

}

export interface JobQuery {
Expand Down
2 changes: 1 addition & 1 deletion app/util/sts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export default class SecureTokenService {
constructor(overrides?: STSClientConfig) {
const endpointSettings: STSClientConfig = {};
if (process.env.USE_LOCALSTACK === 'true') {
endpointSettings.endpoint = 'http://localhost:4592';
endpointSettings.endpoint = `http://${env.localstackHost}:4592`;
}

this.sts = new STSClient({
Expand Down
5 changes: 4 additions & 1 deletion app/views/workflow-ui/job/index.mustache.html
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@
<form id="work-items-query-form" action="./{{job.jobID}}" method="get" class="pt-2">
<input type="hidden" name="jobID" value="{{job.jobID}}" />
<input type="hidden" name="page" value="{{page}}" />
<input type="hidden" name="limit" value="{{limit}}" />
{{> workflow-ui/date-time-picker}}
<input name="tableFilter" class="form-control table-filter mb-2" placeholder="add a filter"
data-value="{{selectedFilters}}" data-is-admin-route="{{isAdminRoute}}">
Expand All @@ -70,6 +69,10 @@
negate statuses
</label>
</div>
<div class="input-group mt-2">
<span class="input-group-text">page size</span>
<input name="limit" type="number" class="form-control" value="{{limit}}">
</div>
<button type="submit" class="btn btn-primary btn-sm mt-3"><i class="bi bi-filter-circle"></i>
apply</button>
</form>
Expand Down
10 changes: 1 addition & 9 deletions app/views/workflow-ui/job/work-items-table.mustache.html
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,4 @@
</tbody>
</table>

<nav aria-label="Page navigation" class="bg-white d-flex flex-column align-items-center py-2 sticky-paging">
<ul class="pagination px-0 mx-auto">
{{#links}}
<li class="page-item {{linkDisabled}}">
<a class="page-link" href="{{linkHref}}" title="{{linkTitle}}">{{linkTitle}}</a>
</li>
{{/links}}
</ul>
</nav>
{{> workflow-ui/paging }}
15 changes: 5 additions & 10 deletions app/views/workflow-ui/jobs/index.mustache.html
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
<div class="col-2">
<form id="jobs-query-form" action="./workflow-ui" method="get" class="pt-2">
<input type="hidden" name="page" value="{{page}}" />
<input type="hidden" name="limit" value="{{limit}}" />
{{> workflow-ui/date-time-picker}}
<input id="sort-granules" type="hidden" name="sortGranules" value="{{sortGranules}}" />
<input name="tableFilter" class="table-filter form-control mb-2" placeholder="add a filter"
Expand All @@ -72,20 +71,16 @@
</label>
</div>
{{/isAdminRoute}}
<div class="input-group mt-2">
<span class="input-group-text">page size</span>
<input name="limit" type="number" class="form-control" value="{{limit}}">
</div>
<button type="submit" class="btn btn-primary btn-sm mt-3"><i class="bi bi-filter-circle"></i> apply</button>
</form>
</div>
<div class="col-10" id="jobs-table-container">
{{> workflow-ui/jobs/jobs-table}}
<nav aria-label="Page navigation" class="bg-white d-flex flex-column align-items-center py-2 sticky-paging">
<ul class="pagination px-0 mx-auto">
{{#links}}
<li class="page-item {{linkDisabled}}">
<a class="page-link" href="{{linkHref}}" title="{{linkTitle}}">{{linkTitle}}</a>
</li>
{{/links}}
</ul>
</nav>
{{> workflow-ui/paging}}
</div>
</div>
</div>
Expand Down
12 changes: 12 additions & 0 deletions app/views/workflow-ui/paging.mustache.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<nav aria-label="Page navigation" class="bg-white d-flex flex-column align-items-center py-2 sticky-paging">
<ul class="pagination px-0 mx-auto mb-1">
{{#links}}
<li class="page-item {{linkDisabled}}">
<a class="page-link" href="{{linkHref}}" title="{{linkTitle}}">{{linkTitle}}</a>
</li>
{{/links}}
</ul>
<small class="text-muted">
{{paginationInfo.from}}-{{paginationInfo.to}} of {{paginationInfo.total}} (page {{paginationInfo.currentPage}} of {{paginationInfo.lastPage}})
</small>
</nav>
2 changes: 2 additions & 0 deletions bin/create-k8s-config-maps-and-secrets
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ data:
USER_AGENT: "${USER_AGENT}"
MAX_DOWNLOAD_RETRIES: "${MAX_DOWNLOAD_RETRIES}"
MAX_PUT_WORK_RETRIES: "${MAX_PUT_WORK_RETRIES}"
WORK_ITEM_UPDATE_QUEUE_URL: "${WORK_ITEM_UPDATE_QUEUE_URL}"
LARGE_WORK_ITEM_UPDATE_QUEUE_URL: "${LARGE_WORK_ITEM_UPDATE_QUEUE_URL}"
ARTIFACT_BUCKET: "${ARTIFACT_BUCKET}"
CLIENT_ID: "${CLIENT_ID}"
CMR_MAX_PAGE_SIZE: "${CMR_MAX_PAGE_SIZE}"
Expand Down
98 changes: 98 additions & 0 deletions bin/port-forward
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/bin/bash
# Script for setting up port forwarding needed for harmony to run locally. Check logs/ directory
# for error messages in setting up port forwards. This script is called from others, not expected
# to be used directly in normal circumstances.

function start_port_forwarding() {
local service_name="$1"
shift 1
local port_pairs=("$@")

local args=()
for port_pair in "${port_pairs[@]}"; do
args+=("${port_pair}")
done

stop_port_forwarding "$service_name"

local max_iterations=60
local iteration=0

while [ "$iteration" -lt "$max_iterations" ]; do
# Wait until there is exactly 1 pod running for the service
local running_count
running_count=$(kubectl get pods -n harmony -l app="$service_name" --no-headers | wc -l)

if [ "$running_count" -eq 1 ]; then
echo "Exactly 1 ${service_name} pod found. Setting up port forward when pod is ready."
break
fi

((iteration++))

sleep 1
done

if [ "$iteration" -eq "$max_iterations" ]; then
echo "WARNING: Timeout reached. There are ${running_count} pods running. Port forwarding may not work."
exit 1
fi

kubectl wait -n harmony --for=condition=ready pod -l app="$service_name" --timeout=60s
nohup kubectl -n harmony port-forward "service/${service_name}" "${args[@]}" > "logs/port-forward-${service_name}.log" 2>&1 &
echo "Port forwarding started for service: ${service_name}, port pairs: ${port_pairs[*]}"
}

function stop_port_forwarding() {
local service_name="$1"
local process_name="kubectl -n harmony port-forward service/${service_name}"
local running=$(pgrep -f "${process_name}")
if [ -n "${running}" ]; then
echo "Stopping port forwarding for service: ${service_name}"
pkill -f "${process_name}"
echo "Port forwarding stopped for service: ${service_name}"
else
echo "No port forwarding running for service: ${service_name}"
fi
}

function is_port_forwarding_running() {
local service_name="$1"
local process_name="kubectl -n harmony port-forward service/${service_name}"
pgrep -f "${process_name}" >/dev/null
}

function print_usage() {
echo "Usage: $0 [start|stop|status] <service-name>"
echo "Commands:"
echo " start Start port forwarding for the specified service"
echo " stop Stop port forwarding for the specified service"
echo " status Check if port forwarding is running for the specified service"
}

if [ "$#" -lt 2 ]; then
print_usage && exit 1
fi

command="$1"
service="$2"

case "$command" in
start)
start_port_forwarding "$service" "${@:3}"
;;
stop)
stop_port_forwarding "$service"
;;
status)
if is_port_forwarding_running "$service"; then
echo "Port forwarding is running for service: $service"
else
echo "Port forwarding is not running for service: $service"
fi
;;
*)
print_usage
exit 1
;;
esac
1 change: 1 addition & 0 deletions bin/start-harmony
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ set +a
eval "$env_save"

envsubst < ./config/harmony-k8s.yaml | kubectl apply -f - -n harmony > /dev/null
bin/port-forward start harmony 3000:3000
echo 'Harmony has started at http://localhost:3000/'
13 changes: 6 additions & 7 deletions bin/start-postgres-localstack
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,12 @@ fi

WORK_ITEM_UPDATE_QUEUE=${WORK_ITEM_UPDATE_QUEUE:-work-item-update-queue.fifo}
LARGE_WORK_ITEM_UPDATE_QUEUE=${LARGE_WORK_ITEM_UPDATE_QUEUE:-large-work-item-update-queue.fifo}
WORK_ITEM_SCHEDULER_QUEUE=${WORK_ITEM_SCHEDULER_QUEUE:-work-item-scheduler-queue.fifo}
WORK_ITEM_SCHEDULER_QUEUE=${WORK_ITEM_SCHEDULER_QUEUE:-work-item-scheduler-queue}
STAGING_BUCKET=${STAGING_BUCKET:-local-staging-bucket}
ARTIFACT_BUCKET=${ARTIFACT_BUCKET:-local-artifact-bucket}
UPLOAD_BUCKET=${UPLOAD_BUCKET:-local-upload-bucket}

# Creates qeueue and buckets in localstack
# Creates queue and buckets in localstack
localstack_startup_script="
import boto3
s3 = boto3.client('s3', endpoint_url='http://localhost:4566', region_name='us-west-2')
Expand All @@ -93,7 +93,7 @@ s3.put_bucket_acl(Bucket='${ARTIFACT_BUCKET}', ACL='public-read') # allows us to
sqs = boto3.client('sqs', endpoint_url='http://localhost:4566', region_name='us-west-2')
sqs.create_queue(QueueName='${WORK_ITEM_UPDATE_QUEUE}', Attributes={'FifoQueue': 'true', 'ContentBasedDeduplication': 'true'})
sqs.create_queue(QueueName='${LARGE_WORK_ITEM_UPDATE_QUEUE}', Attributes={'VisibilityTimeout': '3600', 'FifoQueue': 'true', 'ContentBasedDeduplication': 'true'})
sqs.create_queue(QueueName='${WORK_ITEM_SCHEDULER_QUEUE}', Attributes={'FifoQueue': 'true', 'ContentBasedDeduplication': 'true'})
sqs.create_queue(QueueName='${WORK_ITEM_SCHEDULER_QUEUE}')
"

# For some develpers using Linux, artifact repositories were not working correctly without
Expand All @@ -109,11 +109,10 @@ kubectl -n harmony create configmap localstack-config --from-literal=startup.py=
kubectl apply -n harmony -f ./config/local-postgres-localstack-deployment.yml

[[ $USE_LOCALSTACK = 'true' ]] && s3_endpoint="localstack:4572" || s3_endpoint="s3.amazonaws.com"
POSTGRES_PORT="${POSTGRES_PORT:-5432}"

if [[ "$KUBE_CONTEXT" == "minikube" ]]; then
# Allow LoadBalancer services to be reached in minikube
minikube tunnel > /dev/null &
fi
bin/port-forward start localstack 4566:4566 4572:4566 4592:4566
bin/port-forward start postgres ${POSTGRES_PORT}:5432

echo ''
echo 'Localstack has started at http://localhost:4566/'
Expand Down
15 changes: 15 additions & 0 deletions bin/stop-harmony-and-services
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/bin/bash
# This script will delete all of the kubernetes harmony resources. If you use this script
# you will also delete all of your local harmony jobs since the database will be destroyed.

current_context=$(kubectl config current-context)
if [ "$current_context" != "docker-desktop" ] && [ "$current_context" != "minikube" ]; then
echo 'ERROR: Attempting to use a non-local k8s context while deleting harmony resources.'
echo "$current_context"
exit 1
fi

bin/port-forward stop harmony
bin/port-forward stop localstack
bin/port-forward stop postgres
kubectl delete ns harmony
Loading

0 comments on commit 61fc0a6

Please sign in to comment.