Define a topological order for all crates (#697)

And use it in `scripts/upgrade.sh` to update the `Cargo.toml` file in
the right order.

In the end it didn't happen to be the dependency resolution problem, but
it's still useful to have a way to iterate over crates in topological
order.

scripts/package.sh

@@ -27,6 +27,43 @@ package_bin_path() { _package_bin_string path; }
 
 cargo_info_version() { _cargo_info "$1" version; }
 
+# Tested by ./scripts/publish.sh --dry-run
+TOPOLOGICAL_ORDER='
+one-of
+logger
+wire-derive
+error
+wire
+wire/fuzz
+sync
+protocol
+interpreter
+store
+store/fuzz
+api-desc
+api-desc/crates/update
+api-macro
+api
+stub
+prelude
+board
+scheduler
+protocol-tokio
+protocol-usb
+cli-tools
+cli
+xtask
+protocol/crates/schema
+runner-host/crates/web-common
+runner-host/crates/web-client
+runner-host/crates/web-server
+runner-host
+runner-nordic/crates/header
+runner-nordic
+runner-nordic/crates/bootloader
+wasm-bench
+'
+
 # Internal helpers
 _package_raw() { sed -n '/^\[package]$/,/^$/{s/^'"$1"' = //p}' Cargo.toml; }
 _package_string() { _package_raw "$1" | sed 's/^"\(.*\)"$/\1/'; }

scripts/publish.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 # Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,65 +16,40 @@
 set -e
 . scripts/log.sh
 . scripts/package.sh
+. scripts/test-helper.sh
 
 # This script publishes all crates.
 
-[ -z "$(git status -s)" ] || e "Repository is not clean"
-
-TOPOLOGICAL_ORDER=(
-  one-of
-  logger
-  wire-derive
-  error
-  wire
-  sync
-  protocol
-  interpreter
-  store
-  api-desc
-  api-macro
-  api
-  stub
-  prelude
-  board
-  scheduler
-  protocol-tokio
-  protocol-usb
-  cli-tools
-  cli
-)
-
-listed_crates() {
-  echo "${TOPOLOGICAL_ORDER[@]}" | sed 's/ /\n/g' | sort
-}
-
-published_crates() {
-  find crates -name CHANGELOG.md -printf '%h\n' | sed 's#^crates/##' | sort
-}
+diff_sorted TOPOLOGICAL_ORDER \
+  "$(git ls-files '*/Cargo.toml' | sed -n 's#^crates/\(.*\)/Cargo.toml$#\1#p' | sort)" \
+  $(echo $TOPOLOGICAL_ORDER | sed 's/ /\n/g' | sort)
 
 dependencies() {
-  sed -n 's#^wasefire-.*path = "../\([a-z-]*\)".*$#\1#p' crates/$1/Cargo.toml
+  sed -n 's#^.*path = "\([^"]*\)".*$#\1#p' crates/$1/Cargo.toml | \
+    while read i; do
+      [ ${i%.rs} = $i ] || continue
+      realpath --relative-base=crates -m crates/$1/$i
+    done
 }
 
 occurs_before() {
-  for x in "${TOPOLOGICAL_ORDER[@]}"; do
+  for x in $TOPOLOGICAL_ORDER; do
     [ $x = $1 ] && return
     [ $x = $2 ] && return 1
   done
   return 2
 }
 
-diff <(listed_crates) <(published_crates) \
-  || e 'Listed and published crates do not match (see diff above)'
-
-for crate in "${TOPOLOGICAL_ORDER[@]}"; do
-  for name in $(dependencies $crate); do
-    occurs_before $name $crate || e "$crate depends on $name but occurs before"
+for crate in $TOPOLOGICAL_ORDER; do
+  for dep in $(dependencies $crate); do
+    occurs_before $dep $crate || e "$crate depends on $dep but occurs before"
   done
 done
 
 [ "$1" = --dry-run ] && d "Nothing more to do with --dry-run"
 
+[ -z "$(git status -s)" ] || e "Repository is not clean"
+
 i "Remove all -git suffixes (if any) and reset CHANGELOG tests"
 sed -i 's/-git//' $(git ls-files '*/'{Cargo.{toml,lock},CHANGELOG.md})
 sed -i 's/\(^<!-- .* test\): [0-9]* -->$/\1: 0 -->/' $(git ls-files '*/CHANGELOG.md')
@@ -90,8 +65,9 @@ git log -1 --pretty=%s | grep -q '^Release all crates (#[0-9]*)$' \
   || e "This is not a merged release commit"
 [ "$1" = --no-dry-run ] || d "Run with --no-dry-run to actually publish"
 
-for crate in "${TOPOLOGICAL_ORDER[@]}"; do
+for crate in $TOPOLOGICAL_ORDER; do
   ( cd crates/$crate
+    $(package_publish) || continue
     current="$(package_version)"
     latest="$(cargo_info_version "$(package_name)")"
     if [ "$current" = "$latest" ]; then

scripts/upgrade.sh

@@ -38,8 +38,18 @@ for crate in $(get_crates); do
   update_crate "$crate" "$(cargo_info_version "$crate")"
 done
 
+# TODO(https://github.com/rust-lang/cargo/issues/10307): Remove the loop and inline.
+update_breaking() {
+  while ! x cargo -Z unstable-options update --manifest-path=$1 --breaking; do
+    t 'Manually fix the issue with `cargo update <spec>` and hit ENTER'
+    read garbage
+  done
+}
+for crate in $TOPOLOGICAL_ORDER; do
+  update_breaking crates/$crate/Cargo.toml
+done
 for path in $(git ls-files '*/Cargo.toml'); do
-  cargo -Z unstable-options update --manifest-path=$path --breaking
+  update_breaking $path
 done
 
 ( cd examples/assemblyscript
@@ -49,4 +59,6 @@ ASC_VERSION=$(sed -n 's/^  "version": "\(.*\)",$/\1/p' \
   examples/assemblyscript/node_modules/assemblyscript/package.json)
 x sed -i "/ASC_VERSION:/s/\"[^\"]*\"/\"$ASC_VERSION\"/" crates/xtask/src/main.rs
 
-d "All dependencies have been upgraded"
+x git commit -am'Upgrade all dependencies'
+
+d "All dependencies have been upgraded and a commit created"