Dev hoover 18

.github/workflows/all-builds.yml

@@ -45,9 +45,6 @@ jobs:
           soil/worker.sh JOB-raw-vm
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           soil/github-actions.sh publish-and-exit raw-vm T
 
@@ -85,9 +82,6 @@ jobs:
           soil/github-actions.sh run-job dummy podman
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           soil/github-actions.sh publish-and-exit dummy T
 
@@ -412,9 +406,6 @@ jobs:
           soil/worker.sh JOB-maybe-merge
 
       - name: publish-html
-        env:
-          # for deploying to dashboard
-          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
         run: |
           # NOTE: does not publish to status API
           soil/github-actions.sh publish-and-exit maybe-merge

.github/workflows/fast-subset.yml_DISABLED

@@ -0,0 +1,122 @@
+# Soil wrapped in Github Actions.
+
+name: oil
+on:
+  # We are running into the pitfall here
+  # https://fusectore.dev/2022/09/25/github-actions-pitfalls.html
+  # We only run for members now, not external contributors
+  # But I think their solution of push: branches: [soil-staging] would prevent
+  # us from testing dev / feature branches.  We would have to create a PR
+  # first?
+  pull_request:
+  # Run on PR merge to soil-staging, so that it will get auto-merged to master
+  push:
+    branches: ['soil-staging']
+  #push:
+  #  branches: ['soil-staging', 'dev*', 'jesse*']
+  #
+  # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet
+  # I think that is sufficient for dev branches.
+
+  # don't run CI on master because we test soil-staging and auto-merge.
+  #push:
+  #  branches-ignore:
+  #    - master
+env:
+  # Only set for PR runs.
+  # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request
+  GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+  GITHUB_PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+  GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+jobs:
+  # The perf tool depends on a specific version of a kernel, so run it outside
+  # a container.
+  raw-vm:
+    runs-on: ubuntu-22.04
+    env:
+      REPO_ROOT: ${{ github.workspace }}
+    needs: ['cpp-tarball']
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: raw-vm
+        run: |
+          soil/worker.sh JOB-raw-vm
+
+      - name: publish-html
+        run: |
+          soil/github-actions.sh publish-and-exit raw-vm T
+
+  dummy:
+    runs-on: ubuntu-22.04
+    # container: oilshell/soil-dummy
+    env:
+      REPO_ROOT: ${{ github.workspace }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+        # UPGRADED to podman
+      - name: dummy
+        run: |
+          soil/github-actions.sh run-job dummy podman
+
+      - name: publish-html
+        run: |
+          soil/github-actions.sh publish-and-exit dummy T
+
+  cpp-tarball:
+    runs-on: ubuntu-22.04
+    env:
+      REPO_ROOT: ${{ github.workspace }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Fix kernel mmap rnd bits
+      # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with
+      # high-entropy ASLR in much newer kernels that GitHub runners are
+      # using leading to random crashes: https://reviews.llvm.org/D148280
+        run: sudo sysctl vm.mmap_rnd_bits=28
+
+      - name: cpp-tarball
+        run: |
+          soil/github-actions.sh run-job cpp-tarball
+
+      # can't be done inside container
+      - name: publish-cpp-tarball
+        env:
+          # auth for web server
+          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
+        run: |
+          soil/github-actions.sh publish-cpp-tarball
+
+      - name: publish-html
+        env:
+          OILS_GITHUB_KEY: ${{ secrets.OILS_GITHUB_KEY }}
+        run: |
+          soil/github-actions.sh publish-and-exit cpp-tarball T
+
+  maybe-merge-to-master:
+    runs-on: ubuntu-22.04
+    env:
+      REPO_ROOT: ${{ github.workspace }}
+    # List of tasks to wait on.  Copied from soil/worker.sh list-jobs
+    needs: ['dummy', 'cpp-tarball', 'raw-vm']
+    #needs: ['dummy', 'pea', 'other-tests']
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: maybe-merge
+        env:
+          SOIL_GITHUB_API_TOKEN: ${{ secrets.SOIL_GITHUB_API_TOKEN }}
+        run: |
+          # STUBBED OUT
+          soil/worker.sh JOB-dummy
+
+      - name: publish-html
+        run: |
+          # NOTE: does not publish to status API
+          soil/github-actions.sh publish-and-exit maybe-merge

doc/release-quality.md

@@ -66,7 +66,7 @@ This is a supplement to the [main release page](index.html).
 - [osh-usage](more-tests.wwz/suite-logs/osh-usage.txt).  Misc tests of the `osh` binary.
 - [tools-deps](more-tests.wwz/suite-logs/tools-deps.txt).  Tests for a subcommand in
   progress.
-- [syscall](more-tests.wwz/syscall/-wwz-index) How many syscalls do we make,
+- [syscall](more-tests.wwz/syscall/-wwz-index). How many syscalls do we make,
   and how many processes do we start?
 - [ysh-ify Tests](more-tests.wwz/suite-logs/ysh-ify.txt).  Test OSH to YSH
   translation.

soil/README.md

@@ -29,7 +29,7 @@ Continuous testing on many platforms.
           cpp-small.{tsv,json}
         commits/
 
-      srht-jobs/
+      sourcehut-jobs/
         index.html
         raw.html
         345/  # JOB_ID
@@ -106,7 +106,7 @@ TODO:
     - github-jobs/tmp-$$.{index,raw}.html - shell script does mv
     - github-jobs/commits/tmp-$$.$HASH.html - shell script does mv
       - this is based on github-jobs/$RUN/*.tsv -- similar to format-wwz-index
-      - or srht-jobs/*/*.tsv and filtered by commit
+      - or sourcehut-jobs/*/*.tsv and filtered by commit
     - github-jobs/tmp-$$.remove.txt - shell script does rm
   - status-api/github-jobs/$RUN/$job -- PUT this
 

soil/admin.sh

@@ -15,5 +15,14 @@ keygen() {
   ssh-keygen -t rsa -b 4096 -C "$comment" -f $file
 }
 
+enable-fast-subset() {
+  git mv .github/workflows/all-builds.yml{,_DISABLED}
+  git mv .github/workflows/fast-subset.yml{_DISABLED,}
+}
+
+disable-fast-subset() {
+  git mv .github/workflows/all-builds.yml{_DISABLED,}
+  git mv .github/workflows/fast-subset.yml{,_DISABLED}
+}
 
 "$@"

soil/common.sh

@@ -22,9 +22,9 @@ dump-env() {
 
 if true; then
   readonly SOIL_USER='travis_admin'
-  readonly SOIL_HOST='travis-ci.oilshell.org'
-  readonly SOIL_HOST_DIR=~/travis-ci.oilshell.org  # used on server
-  readonly SOIL_REMOTE_DIR=travis-ci.oilshell.org  # used on client
+  readonly SOIL_HOST='ci.oilshell.org'
+  readonly SOIL_HOST_DIR=~/ci.oilshell.org  # used on server
+  readonly SOIL_REMOTE_DIR=ci.oilshell.org  # used on client
 elif false; then
   readonly SOIL_USER='oils'
   readonly SOIL_HOST='mb.oils.pub'
@@ -40,6 +40,8 @@ fi
 
 readonly SOIL_USER_HOST="$SOIL_USER@$SOIL_HOST"
 
+readonly WWUP_URL="https://$SOIL_HOST/uuu/wwup.cgi"
+
 html-head() {
   # TODO: Shebang line should change too
   PYTHONPATH=. python3 doctools/html_head.py "$@"
@@ -73,7 +75,7 @@ git-commit-dir() {
   local commit_hash
   commit_hash=$(cat _tmp/soil/commit-hash.txt)
 
-  local git_commit_dir="$SOIL_REMOTE_DIR/${prefix}jobs/git-$commit_hash"
+  local git_commit_dir="$SOIL_REMOTE_DIR/code/${prefix}jobs/git-$commit_hash"
 
   echo $git_commit_dir
 }
@@ -85,8 +87,8 @@ git-commit-url() {
   local commit_hash
   commit_hash=$(cat _tmp/soil/commit-hash.txt)
 
-  # https:// not working on Github Actions?
-  local url="http://$SOIL_HOST/${prefix}jobs/git-$commit_hash"
+  # https:// not working on Github Actions because of cert issues?
+  local url="http://$SOIL_HOST/code/${prefix}jobs/git-$commit_hash"
 
   echo $url
 }

soil/github-actions.sh

@@ -27,6 +27,9 @@ publish-html-assuming-ssh-key() {
     # Recommended by the docs
     export JOB_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
 
+    # Note $GITHUB_RUN_NUMBER is a different sequence for all-builds.yml vs.
+    # fast-subset.yml
+
     soil/web-worker.sh deploy-job-results 'github-' $GITHUB_RUN_NUMBER $job_name \
       JOB_URL \
       GITHUB_WORKFLOW	\
@@ -85,7 +88,7 @@ load-secret-key() {
 publish-html() {
   ### Publish job HTML, and optionally status-api
 
-  load-secret-key
+  #load-secret-key
 
   set -x
   # $1 can be the job name

soil/maybe-merge.sh

@@ -125,7 +125,7 @@ soil-run() {
   mkdir -p $dir
 
   # These tiny files are written by each Soil task
-  local url_base="http://$SOIL_HOST/status-api/github/$run_id"
+  local url_base="http://$SOIL_HOST/uuu/status-api/github/$run_id"
 
   #local jobs='dummy pea other-tests'  # minimal set of jobs to wait for
   local jobs=$(soil/worker.sh list-jobs)

soil/sourcehut.sh

@@ -28,7 +28,7 @@ publish-html-assuming-ssh-key() {
   local job_name=$1
 
   if true; then
-    soil/web-worker.sh deploy-job-results 'srht-' $JOB_ID $job_name JOB_ID JOB_URL
+    soil/web-worker.sh deploy-job-results 'sourcehut-' $JOB_ID $job_name JOB_ID JOB_URL
   else
     soil/web-worker.sh deploy-test-wwz  # dummy data that doesn't depend on the build
   fi
@@ -40,12 +40,12 @@ publish-html-assuming-ssh-key() {
   # Note: the directory structure will be overlapping, unlike Github which has
   # GITHUB_RUN_NUMBER
   #
-  # srht-jobs/
+  # sourcehut-jobs/
   #   1234/foo.wwz  # individual jobs
   #   1235/bar.wwz
   #   git-0101abab/index.html  # commit hash
 
-  time soil/web-worker.sh remote-event-job-done 'srht-' "git-$commit_hash"
+  time soil/web-worker.sh remote-event-job-done 'sourcehut-' "git-$commit_hash"
 }
 
 #

soil/web-init.sh

@@ -26,7 +26,7 @@ source soil/common.sh  # for SOIL_USER and SOIL_HOST
 home-page() {
   ### travis-ci.oilshell.org home page
 
-  local domain=${1:-'travis-ci.oilshell.org'}
+  local domain=${1:-$SOIL_HOST}
   local title="Soil on $domain"
   soil-html-head "$title"
 
@@ -53,24 +53,31 @@ home-page() {
 
       <tr>
         <td>
-          <a href="srht-jobs/">sr.ht</a> 
+          <a href="uuu/sourcehut-jobs/">sr.ht</a> 
         </td>
         <td>
           <a href="https://builds.sr.ht/~andyc">builds.sr.ht</a>
         </td>
-        <td></td>
+        <td>
+          <a href="https://github.com/oils-for-unix/oils/tree/master/.builds">.builds</a>
+        </td>
       </tr>
 
       <tr>
         <td>
-          <a href="github-jobs/">Github Actions</a> 
+          <a href="uuu/github-jobs/">Github Actions</a> 
         </td>
         <td>
           <a href="https://github.com/oilshell/oil/actions/workflows/all-builds.yml">github.com</a>
         </td>
-        <td></td>
+        <td>
+          <a href="https://github.com/oils-for-unix/oils/tree/master/.github/workflows">.github/workflows</a>
+        </td>
       </tr>
+EOF
 
+  if false; then
+    echo '
       <tr>
         <td>
           <a href="circle-jobs/">Circle CI</a> 
@@ -100,40 +107,46 @@ home-page() {
         </td>
         <td></td>
       </tr>
+      '
+  fi
 
+  echo '
     </table>
 
     <h1>Links</h1>
 
     <ul>
       <li>
-        <a href="status-api/github/">static-api/github/</a>
+        <a href="code/github-jobs/">code/github-jobs/</a> - tarballs at every commit
+      </li>
+      <li>
+        <a href="uuu/status-api/github/">uuu/static-api/github/</a> - files used by the CI
       </li>
     </ul>
 
   </body>
 </html>
-EOF
+'
 }
 
 deploy-data() {
   local user=${1:-$SOIL_USER}
   local host=${2:-$SOIL_HOST}
 
-  # www/ prefix for Mythic beasts
-  local host_dir=$SOIL_REMOTE_DIR
+  local host_dir=$SOIL_REMOTE_DIR/uuu
 
   # TODO: Better to put HTML in www/$host/uuu/github-jobs, etc.
   ssh $user@$host mkdir -v -p \
-    $host_dir/{travis-jobs,srht-jobs,github-jobs,circle-jobs,cirrus-jobs,web,status-api/github} \
+    $host_dir/{sourcehut-jobs,github-jobs,status-api/github} \
     $host_dir/web/table
 
-  home-page "$host" > _tmp/index.html
-
   # note: duplicating CSS
-  scp _tmp/index.html $user@$host:$host_dir/
   scp web/{base.css,soil.css,ajax.js} $user@$host:$host_dir/web
   scp web/table/*.{js,css} $user@$host:$host_dir/web/table
+
+  home-page "$host" > _tmp/index.html
+  # Home page goes in the domain root
+  scp _tmp/index.html $user@$host:$SOIL_REMOTE_DIR/
 }
 
 soil-web-manifest() {