Skip to content

Commit

Permalink
convert : fix Norway problem when parsing YAML (#12114)
Browse files Browse the repository at this point in the history
* convert : fix Norway problem when parsing YAML

* Update gguf-py/gguf/metadata.py

* add newline at correct place
  • Loading branch information
ngxson authored Feb 28, 2025
1 parent 70680c4 commit 06c2b15
Showing 1 changed file with 28 additions and 8 deletions.
36 changes: 28 additions & 8 deletions gguf-py/gguf/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,19 +121,39 @@ def load_model_card(model_path: Optional[Path] = None) -> dict[str, Any]:
if not model_card_path.is_file():
return {}

# The model card metadata is assumed to always be in YAML
# The model card metadata is assumed to always be in YAML (frontmatter)
# ref: https://github.com/huggingface/transformers/blob/a5c642fe7a1f25d3bdcd76991443ba6ff7ee34b2/src/transformers/modelcard.py#L468-L473
yaml_content: str = ""
with open(model_card_path, "r", encoding="utf-8") as f:
if f.readline() == "---\n":
raw = f.read().partition("---\n")[0]
data = yaml.safe_load(raw)
if isinstance(data, dict):
return data
content = f.read()
lines = content.splitlines()
lines_yaml = []
if len(lines) == 0:
# Empty file
return {}
if len(lines) > 0 and lines[0] != "---":
# No frontmatter
return {}
for line in lines[1:]:
if line == "---":
break # End of frontmatter
else:
logger.error(f"while reading YAML model card frontmatter, data is {type(data)} instead of dict")
return {}
lines_yaml.append(line)
yaml_content = "\n".join(lines_yaml) + "\n"

# Quick hack to fix the Norway problem
# https://hitchdev.com/strictyaml/why/implicit-typing-removed/
yaml_content = yaml_content.replace("- no\n", "- \"no\"\n")

if yaml_content:
data = yaml.safe_load(yaml_content)
if isinstance(data, dict):
return data
else:
logger.error(f"while reading YAML model card frontmatter, data is {type(data)} instead of dict")
return {}
else:
return {}

@staticmethod
def load_hf_parameters(model_path: Optional[Path] = None) -> dict[str, Any]:
Expand Down

0 comments on commit 06c2b15

Please sign in to comment.