From 022db027dafb68f57e6769dfa75f9fdd36581903 Mon Sep 17 00:00:00 2001 From: Prajwal S N Date: Thu, 27 Jun 2024 23:28:03 +0200 Subject: [PATCH] chore(magicgen): migrate script to Python The old script was in Go, and naively split the lines by whitespace to identify the magic names and values. The new Python script uses a proper regex rule to parse the input and generate the output. Signed-off-by: Prajwal S N --- scripts/magic-gen/Makefile | 7 +- scripts/magic-gen/magicgen-test.sh | 10 +-- scripts/magic-gen/magicgen.go | 110 ----------------------------- scripts/magic-gen/magicgen.py | 82 +++++++++++++++++++++ scripts/magic-gen/magicgen_test.go | 110 ----------------------------- 5 files changed, 87 insertions(+), 232 deletions(-) delete mode 100644 scripts/magic-gen/magicgen.go create mode 100644 scripts/magic-gen/magicgen.py delete mode 100644 scripts/magic-gen/magicgen_test.go diff --git a/scripts/magic-gen/Makefile b/scripts/magic-gen/Makefile index f0bbeace3..eb6ad367e 100644 --- a/scripts/magic-gen/Makefile +++ b/scripts/magic-gen/Makefile @@ -1,17 +1,14 @@ -GO ?= go +PY ?= python3 MAGIC_DEST ?= ../../magic/magic.go magic-gen: clean magic.h - $(GO) run magicgen.go magic.h $(MAGIC_DEST) + @$(PY) magicgen.py magic.h $(MAGIC_DEST) magic.h: curl -s https://raw.githubusercontent.com/checkpoint-restore/criu/criu-dev/criu/include/magic.h -o magic.h test: - @echo "Running unit test" - $(GO) test -v - @echo "Running E2E test" @./magicgen-test.sh @rm -f input.h output.go expected.go diff --git a/scripts/magic-gen/magicgen-test.sh b/scripts/magic-gen/magicgen-test.sh index 2c3e105ea..c5ba6a49b 100755 --- a/scripts/magic-gen/magicgen-test.sh +++ b/scripts/magic-gen/magicgen-test.sh @@ -29,18 +29,14 @@ func LoadMagic() MagicMap { } EOF -if [ -n "$GOCOVERDIR" ]; then - export LOCALFLAGS="-cover" -fi - -go run $LOCALFLAGS magicgen.go input.h output.go +python3 magicgen.py input.h output.go cmp output.go expected.go if [[ $? -eq 0 ]] then - echo "PASS" + echo "---PASS---" exit 0 else - echo "FAIL" + echo "---FAIL---" diff output.go expected.go exit 1 fi diff --git a/scripts/magic-gen/magicgen.go b/scripts/magic-gen/magicgen.go deleted file mode 100644 index b71da7374..000000000 --- a/scripts/magic-gen/magicgen.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "io" - "log" - "os" - "sort" - "strconv" - "strings" -) - -func main() { - // Check arguments - if len(os.Args) != 3 { - log.Fatal("Usage: magicgen.go path/to/magic.h path/to/magic.go") - } - - magicHPath := os.Args[1] - magicGoPath := os.Args[2] - // Open magic.h - magicHFile, err := os.Open(magicHPath) - if err != nil { - log.Fatal("Failed to open magic.h") - } - defer magicHFile.Close() - // Open magic.json if it exists, else create it - magicGoFile, err := os.Create(magicGoPath) - if err != nil { - log.Fatal("Failed to open magic.go") - } - defer magicGoFile.Close() - - magics := readMagics(magicHFile) - writeMagics(magicGoFile, magics) -} - -func readMagics(in io.Reader) map[string]uint64 { - magics := make(map[string]uint64) - - r := bufio.NewScanner(in) - // Read line by line - for r.Scan() { - tokens := strings.Fields(r.Text()) - if len(tokens) >= 3 { - if tokens[0] == "#define" { - magicName := tokens[1] - magicHex := tokens[2] - // Check if magic is defined as another magic - if _, ok := magics[magicHex]; ok { - continue - } - /* Base 0 has to be used to parse magicHex instead - of base 16 as the string contains 0x at the - beginning, which strconv.ParseUint cannot parse. */ - magicHexInt, err := strconv.ParseUint(magicHex, 0, 64) - if err != nil { - log.Printf("Failed to parse magic %s\n", magicName) - continue - } - magics[magicName] = magicHexInt - } - } - } - - return magics -} - -func writeMagics(out io.Writer, magics map[string]uint64) { - // Sort the magic names alphabetically and use - // this in the generated file for consistency - magicNames := make([]string, 0, len(magics)) - for name := range magics { - magicNames = append(magicNames, name) - } - sort.Strings(magicNames) - - w := bufio.NewWriter(out) - fmt.Fprintf(w, - `// Code generated by magicgen. DO NOT EDIT. - -package magic - -type MagicMap struct { - ByName map[string]uint64 - ByValue map[uint64]string -} - -func LoadMagic() MagicMap { - magicMap := MagicMap{ - ByName: make(map[string]uint64), - ByValue: make(map[uint64]string), - }`) - - for _, name := range magicNames { - v := magics[name] - // Ignore RAW_IMAGE_MAGIC and V1 magic - if v == 0 || v == 1 { - continue - } - name = strings.Replace(name, "_MAGIC", "", -1) - fmt.Fprintf(w, "\n\tmagicMap.ByName[\"%s\"] = %d", - name, v) - fmt.Fprintf(w, "\n\tmagicMap.ByValue[%d] = \"%s\"", - v, name) - } - fmt.Fprintf(w, "\n\treturn magicMap\n}\n") - w.Flush() -} diff --git a/scripts/magic-gen/magicgen.py b/scripts/magic-gen/magicgen.py new file mode 100644 index 000000000..355551c1c --- /dev/null +++ b/scripts/magic-gen/magicgen.py @@ -0,0 +1,82 @@ +import sys +import re +import argparse + +def main(): + parser = argparse.ArgumentParser( + description='A script to generate Go bindings for the CRIU magic values provided by magic.h') + parser.add_argument( + 'src', help='Input path (magic.h)', type=str) + parser.add_argument( + 'dest', help='Output path (magic.go)', type=str) + + args = parser.parse_args() + + try: + with open(args.src, 'r') as src: + magics = read_magics(src) + except IOError: + sys.exit("Failed to open magic.h") + + try: + with open(args.dest, 'w') as dest: + write_magics(dest, magics) + except IOError: + sys.exit("Failed to open magic.py") + +def read_magics(src): + magics = {} + # The magic.h header file contains lines like below: + # #define MAGIC_NAME MAGIC_VALUE + # where the magic value can be a hexadecimal or regular + # integer value. The regex below matches lines in the + # header and extracts the name and value as regex groups. + pattern = re.compile(r"#define\s+(\S+)\s+(0x[0-9A-Fa-f]+|\d+)") + + for line in src: + match = pattern.match(line) + if match: + name = match.group(1) + value_str = match.group(2) + # If the magic is defined as another magic, skip it + if value_str in magics: + continue + try: + value = int(value_str, 16) + magics[name] = value + except ValueError: + print(f"Failed to parse magic {name}") + continue + + return magics + +def write_magics(dest, magics): + dest.write( + '''// Code generated by magicgen. DO NOT EDIT. + +package magic + +type MagicMap struct { +\tByName map[string]uint64 +\tByValue map[uint64]string +} + +func LoadMagic() MagicMap { +\tmagicMap := MagicMap{ +\t\tByName: make(map[string]uint64), +\t\tByValue: make(map[uint64]string), +\t}''') + + for name in sorted(magics.keys()): + value = magics[name] + # Ignore RAW_IMAGE_MAGIC and V1 magic + if value == 0 or value == 1: + continue + name = name.replace("_MAGIC", "") + dest.write(f'\n\tmagicMap.ByName["{name}"] = {value}') + dest.write(f'\n\tmagicMap.ByValue[{value}] = "{name}"') + + dest.write('\n\treturn magicMap\n}\n') + +if __name__ == "__main__": + main() diff --git a/scripts/magic-gen/magicgen_test.go b/scripts/magic-gen/magicgen_test.go deleted file mode 100644 index cdc761fe1..000000000 --- a/scripts/magic-gen/magicgen_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "reflect" - "strings" - "testing" -) - -type testRead struct { - in string - expected map[string]uint64 -} - -type testWrite struct { - in map[string]uint64 - expected string -} - -func TestReadMagics(t *testing.T) { - tests := []testRead{ - { - // Simple - in: "#define TEST 0xbc614e", - expected: map[string]uint64{ - "TEST": 12345678, - }, - }, - { - // With duplicate - in: `#define TEST 0xbc614e -#define V1 1 -#define TEST_COPY TEST -#define RAW 0x0 -`, - expected: map[string]uint64{ - "V1": 1, - "TEST": 12345678, - "RAW": 0, - }, - }, - } - - for _, test := range tests { - got := readMagics(strings.NewReader(test.in)) - if !reflect.DeepEqual(got, test.expected) { - t.Errorf("Got: %v\nExpected: %v", got, test.expected) - } - } -} - -func TestWriteMagics(t *testing.T) { - expectedPrefix := `// Code generated by magicgen. DO NOT EDIT. - -package magic - -type MagicMap struct { - ByName map[string]uint64 - ByValue map[uint64]string -} - -func LoadMagic() MagicMap { - magicMap := MagicMap{ - ByName: make(map[string]uint64), - ByValue: make(map[uint64]string), - }` - expectedSuffix := "\n\treturn magicMap\n}\n" - - tests := []testWrite{ - { - // Simple - in: map[string]uint64{ - "TEST": 12345678, - }, - expected: ` - magicMap.ByName["TEST"] = 12345678 - magicMap.ByValue[12345678] = "TEST"`, - }, - { - // With suffix - in: map[string]uint64{ - "TEST_MAGIC": 12345678, - }, - expected: ` - magicMap.ByName["TEST"] = 12345678 - magicMap.ByValue[12345678] = "TEST"`, - }, - { - // With values to be ignored - in: map[string]uint64{ - "V1": 1, - "TEST": 12345678, - "RAW": 0, - }, - expected: ` - magicMap.ByName["TEST"] = 12345678 - magicMap.ByValue[12345678] = "TEST"`, - }, - } - - for _, test := range tests { - got := &bytes.Buffer{} - want := fmt.Sprint(expectedPrefix, test.expected, expectedSuffix) - writeMagics(got, test.in) - if !reflect.DeepEqual(got.String(), want) { - t.Errorf("Got: %v\nExpected: %v", got, want) - } - } -}