Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add initial FML generator based on the StructureComparison code #163

Open
wants to merge 4 commits into
base: experimental
Choose a base branch
from
Open

Add initial FML generator based on the StructureComparison code #163

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b46596b
Add initial FML generator based on the StructureComparison code (simi…
brianpos Sep 5, 2024
8836a44
Use the ForMdTable() string formatting function for message columns too
brianpos Sep 5, 2024
af83fb0
Retrieve the Structure Definiton for the source type to use the base …
brianpos Sep 5, 2024
05fcb7f
Not ideal, but exclude properties that came from the base type (but d…
brianpos Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/Microsoft.Health.Fhir.CodeGen/CompareTool/ComparisonClasses.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ public override string GetStatusString()
public record class StructureComparison : ComparisonTopLevelBase<StructureInfoRec>
{
public required Dictionary<string, ElementComparison> ElementComparisons { get; init; }
public required IEnumerable<KeyValuePair<string, ElementDefinition>> UnmappedProperties { get; init; }

public override string GetStatusString()
{
Expand Down
260 changes: 260 additions & 0 deletions src/Microsoft.Health.Fhir.CodeGen/CompareTool/PackageComparer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,17 @@ public void WriteMapFiles(PackageComparison packageComparison, string? outputDir

WriteStructureBasedConceptMaps(mapSubDir, packageComparison.Extensions.Values.SelectMany(l => l.Select(s => s)));
}

string fmlSubDir = Path.Combine(outputDir, "fml");
if (!Directory.Exists(fmlSubDir))
{
Directory.CreateDirectory(fmlSubDir);
}

// WriteStructureMaps(fmlSubDir, packageComparison.PrimitiveTypes.Values.SelectMany(l => l.Select(s => s)));
WriteStructureMaps(fmlSubDir, packageComparison.ComplexTypes.Values.SelectMany(l => l.Select(s => s)));
WriteStructureMaps(fmlSubDir, packageComparison.Resources.Values.SelectMany(l => l.Select(s => s)));
// WriteStructureMaps(fmlSubDir, packageComparison.Extensions.Values.SelectMany(l => l.Select(s => s)));
}

public void WriteCrossVersionExtensionArtifacts(PackageComparison packageComparison, string? outputDir = null)
Expand Down Expand Up @@ -1402,6 +1413,244 @@ private void WriteValueSetMaps(string outputDir, IEnumerable<ValueSetComparison>
}
}

private void WriteStructureMaps(string outputDir, IEnumerable<StructureComparison> values)
{
if (_crossVersion == null)
{
return;
}

foreach (StructureComparison c in values)
{
ConceptMap? cm = _crossVersion.TryGetSourceStructureElementConceptMap(c);
if (cm == null)
{
continue;
}


// only handling maps between canonicals (which should be all of them)
if (!(cm.SourceScope is Canonical) || !(cm.TargetScope is Canonical))
continue;

Canonical srcScope = (Canonical)cm.SourceScope;
Canonical tgtScope = (Canonical)cm.TargetScope;

var srcVersion = FhirReleases.FhirVersionToSequence(srcScope.Version ?? "");
var tgtVersion = FhirReleases.FhirVersionToSequence(tgtScope.Version ?? "");
string? srcResourceType = srcScope.Uri?.Substring(srcScope.Uri.LastIndexOf('/') + 1);
string? tgtResourceType = tgtScope.Uri?.Substring(tgtScope.Uri.LastIndexOf('/') + 1);
string filename = Path.Combine(outputDir, $"StructureMap-{cm.Id}.json");
{
if (srcScope.Uri == tgtScope.Uri)
filename = Path.Combine(outputDir, $"{srcResourceType}{srcVersion.ToRLiteral().TrimStart('R')}to{tgtVersion.ToRLiteral().TrimStart('R')}.json");
}

// Create the StructureMap!
var sm = new StructureMap();
sm.Id = $"{srcResourceType}{srcVersion.ToRLiteral().TrimStart('R')}to{tgtVersion.ToRLiteral().TrimStart('R')}";
sm.Status = PublicationStatus.Draft;
sm.Name = sm.Id;
sm.Url = $"http://hl7.org/fhir/uv/xver/StructureMap/{sm.Id}";
sm.Title = $"{c.Source.Name} Transforms: {srcVersion.ToRLiteral()} to {tgtVersion.ToRLiteral()}";
sm.Description = c.Message;

sm.Import = [$"http://hl7.org/fhir/uv/xver/StructureMap/*{srcVersion.ToRLiteral().TrimStart('R')}to{tgtVersion.ToRLiteral().TrimStart('R')}"];

// Create the uses structures
sm.Structure.Add(new StructureMap.StructureComponent()
{
Mode = StructureMap.StructureMapModelMode.Source,
Alias = srcResourceType + srcVersion.ToRLiteral(),
Url = srcScope.Uri?.Replace("http://hl7.org/fhir/StructureDefinition/", $"http://hl7.org/fhir/{srcVersion.ToShortVersion()}/StructureDefinition/"),
});
sm.Structure.Add(new StructureMap.StructureComponent()
{
Mode = StructureMap.StructureMapModelMode.Target,
Alias = tgtResourceType + tgtVersion.ToRLiteral(),
Url = tgtScope.Uri?.Replace("http://hl7.org/fhir/StructureDefinition/", $"http://hl7.org/fhir/{tgtVersion.ToShortVersion()}/StructureDefinition/"),
});

// Create the groups
var propComparisons = c.ElementComparisons.ToList();
// var propComparisons = c.ElementComparisons.OrderBy(kvp => kvp.Key).ToList();
foreach ((string path, ElementComparison elementComparison) in propComparisons)
{
if (elementComparison.Source.Types.Any(t => t.Value.Name == "BackboneElement") || !elementComparison.Source.Path.Contains("."))
{
var group = new StructureMap.GroupComponent();
group.Name = String.Join("", elementComparison.Source.Path.Split('.').Select(p => p.Substring(0, 1).ToUpper() + p.Substring(1))); // need to convert the path into a better name than this so you get AccountGuarantor
group.Input.Add(new StructureMap.InputComponent()
{
Mode = StructureMap.StructureMapInputMode.Source,
Name = "src",
// Type = elementComparison.Source.Types
});
group.Input.Add(new StructureMap.InputComponent()
{
Mode = StructureMap.StructureMapInputMode.Target,
Name = "tgt",
// Type = elementComparison.Source.Types
});
if (!elementComparison.Source.Path.Contains("."))
{
group.Input[0].Type = srcResourceType + srcVersion.ToRLiteral();
group.Input[1].Type = tgtResourceType + tgtVersion.ToRLiteral();
group.Extends = "Resource";
Copy link
Contributor Author

@brianpos brianpos Sep 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't spot the routine to retrieve the Base type (i.e. all the StructureDefinition content) for the structure being processed in the resource.
(searching for DomainResource or Resource, or the other datatypes where they are for complex types)
Should I just be using _source.TryResolveByCanonicalUri()

if (_source.TryResolveByCanonicalUri(c.Source.Url, out Resource? sd))
        group.Extends = (sd as StructureDefinition)?.cgBaseTypeName();

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the StructureComparison object (c, I believe at this point in the code), there are references to the Source and Target StructureDefinition resources subject to comparison.

In each of the structures, there is the function to get the base type (cgBaseTypeName).

In DefinitionCollection, there are a few functions you could use - I would probably lean towards TryGetStructure.

If we want a collapsed convenience function to do both, I would be fine with that (e.g., cgBaseTypeStructure).

}
if (elementComparison.Source.Types.Any(t => t.Value.Name == "BackboneElement"))
{
group.Extends = "BackboneElement";
}
if (!elementComparison.Source.Types.Any() && !elementComparison.Source.Path.Contains("."))
{
// This is the root type mapping, so can be the default
group.TypeMode = StructureMap.StructureMapGroupTypeMode.TypeAndTypes;
}
sm.Group.Add(group);

// Now scan for the properties at this level in the resource
foreach ((string pathInner, ElementComparison elementComparisonInner) in propComparisons)
{
if (!elementComparisonInner.Source.Path.Contains("."))
continue;
if (elementComparisonInner.Source.Types.Any(t => t.Value.Name == "BackboneElement"))
{
// This is the calling of the backbone element copying part
// continue;
}

if (!elementComparisonInner.Source.Path.StartsWith($"{path}.") || elementComparisonInner.Source.Path.LastIndexOf(".") > elementComparison.Source.Path.Length)
continue;

// Filter out any backbone element props
var backboneElementProps = new[] { "id", "extension", "modifierExtension" };
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There has to be a better way to locate if the property isn't from the differential (and thus not required at this level in the resource)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Today there is a function in ElementDefinitionExtensions, cgIsInherited which tells you whether an element is inherited or not, based on the path.

The function will work well in this scenario (all core definitions), but I have a note to update it for profile detection.

if (elementComparison.Source.Types.Any(t => t.Value.Name == "BackboneElement") && backboneElementProps.Contains(elementComparisonInner.Source.Name))
continue;
var resourceElementProps = new[] { "id", "extension", "modifierExtension", "meta", "contained", "implicitRules", "language", "text" };
if (!elementComparison.Source.Types.Any() && resourceElementProps.Contains(elementComparisonInner.Source.Name))
continue;


if (!elementComparisonInner.TargetMappings.Any())
{
// Add in the rule (for the no map scenario)
var rule = new StructureMap.RuleComponent()
{
Name = elementComparisonInner.Source.Name + "_not_mapped",
Documentation = $"No matching target property detected for {elementComparisonInner.Source.Path}"
};
group.Rule.Add(rule);
// Source
var ruleSource = new StructureMap.SourceComponent()
{
Context = "src",
Element = elementComparisonInner.Source.Name
};
rule.Source.Add(ruleSource);
continue;
}

foreach (var targetMapping in elementComparisonInner.TargetMappings)
{
foreach (var sourceType in elementComparisonInner.Source.Types)
{
// Add in the rule
var rule = new StructureMap.RuleComponent()
{
Name = elementComparisonInner.Source.Name,
};
group.Rule.Add(rule);
// Source
var ruleSource = new StructureMap.SourceComponent()
{
Context = "src",
Element = elementComparisonInner.Source.Name
};
rule.Source.Add(ruleSource);

var ruleTarget = new StructureMap.TargetComponent()
{
Context = "tgt",
Element = targetMapping.Target?.Name
};
rule.Target.Add(ruleTarget);

// Check if there are multiple types encountered
if (elementComparisonInner.Source.Types.Count > 1)
{
ruleSource.Element = ruleSource.Element.Replace("[x]", "");
ruleSource.Type = sourceType.Key;
ruleTarget.Element = ruleTarget.Element?.Replace("[x]", "");
rule.Name = rule.Name.Replace("[x]", sourceType.Key.Substring(0,1).ToUpper() + sourceType.Key.Substring(1));

// Also check that this type is in the target property
if (!targetMapping.TypeComparisons.ContainsKey(sourceType.Key) || targetMapping.TypeComparisons[sourceType.Key]?.Relationship != CMR.Equivalent)
rule.Documentation = $"Type mapping issue found for {sourceType.Key} - {targetMapping.TypeComparisons[sourceType.Key]?.Relationship}\n{elementComparisonInner.Message}";
}

if (elementComparisonInner.Source.Types.Any(t => t.Value.Name == "BackboneElement"))
{
// This is the calling of the backbone element copying part
ruleSource.Variable = "s";
ruleTarget.Variable = "t";
rule.Dependent.Add(new StructureMap.DependentComponent()
{
Name = String.Join("", elementComparisonInner.Source.Path.Split('.').Select(p => p.Substring(0, 1).ToUpper() + p.Substring(1))),
});
rule.Dependent[0].Parameter.Add(new StructureMap.ParameterComponent() { Value = new FhirString("s") });
rule.Dependent[0].Parameter.Add(new StructureMap.ParameterComponent() { Value = new FhirString("t") });
}
if (targetMapping.Relationship != CMR.Equivalent)
{
if (!string.IsNullOrEmpty(rule.Documentation))
rule.Documentation += "\n";
rule.Documentation += elementComparisonInner.Message;
}
}
}
}

// Report a list of properties that are NOT mapped from the target
foreach (var prop in c.UnmappedProperties.Where(ump => ump.Value.Path.StartsWith(elementComparison.Source.Path)))
{
if (!prop.Value.Path.StartsWith($"{path}.") || prop.Value.Path.LastIndexOf(".") > elementComparison.Source.Path.Length)
continue;

// Add in the rule (for the no map scenario)
var rule = new StructureMap.RuleComponent()
{
Name = "tgt_" + prop.Value.Path.Replace(elementComparison.Source.Path+".", "") + "_no_source",
Documentation = $"No matching source property detected for {prop.Value.Path}"
};
group.Rule.Add(rule);
// Source
var ruleSource = new StructureMap.SourceComponent()
{
Context = "src"
};
rule.Source.Add(ruleSource);
}
}
}

try
{
var fmlText = Hl7.Fhir.MappingLanguage.StructureMapUtilitiesParse.render(sm);
System.IO.File.WriteAllText(filename.Replace(".json", ".fml"), fmlText);
//using FileStream fs = new(filename, FileMode.Create, FileAccess.Write);
//using Utf8JsonWriter writer = new(fs, new JsonWriterOptions() { Indented = true, });
//{
// JsonSerializer.Serialize(writer, sm, _firelySerializerOptions);
//}
}
catch (Exception ex)
{
Console.WriteLine($"Error writing {filename}: {ex.Message} {ex.InnerException?.Message}");
}
}
}

private void WriteStructureBasedConceptMaps(string outputDir, IEnumerable<StructureComparison> values)
{
if (_crossVersion == null)
Expand Down Expand Up @@ -2582,6 +2831,7 @@ private Dictionary<string, List<StructureComparison>> CompareStructures(
Relationship = null,
Message = $"{e.Path} does not exist in {_targetRLiteral} and has no mapping",
}).ToDictionary(e => e.Source.Path),
UnmappedProperties = [],
});
}
}
Expand Down Expand Up @@ -2785,6 +3035,9 @@ internal bool TryCompareStructureElements(

CMR? sdRelationship = RelationshipForComparisons(elementComparisons);

// see what was left over "untouched" in the target
IEnumerable<KeyValuePair<string, ElementDefinition>> unmappedTargetProperties = targetElements.Where(te => !targetsMappedToSources.ContainsKey(te.Key));

comparison = new()
{
Source = GetInfo(sourceSd),
Expand All @@ -2793,7 +3046,14 @@ internal bool TryCompareStructureElements(
ElementComparisons = elementComparisons,
Relationship = sdRelationship,
Message = MessageForComparisonRelationship(sdRelationship, sourceSd, targetSd),
UnmappedProperties = unmappedTargetProperties
};

if (unmappedTargetProperties.Any())
{
comparison = comparison with { Message = comparison.Message + " - unmapped " + String.Join(", ", unmappedTargetProperties.Select(utp => utp.Key)) };
}

return true;


Expand Down
3 changes: 2 additions & 1 deletion src/Microsoft.Health.Fhir.CodeGen/Microsoft.Health.Fhir.CodeGen.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

<ItemGroup>
<PackageReference Include="brianpos.Fhir.Base.FhirPath.Validator" Version="5.9.0-rc1" />
<PackageReference Include="brianpos.Fhir.R5.MappingLanguage" Version="5.9.0-beta3" />
<PackageReference Include="Firely.Fhir.Packages" Version="4.7.0" />
<PackageReference Include="Hl7.Fhir.R5" Version="5.9.0" />
<PackageReference Include="Microsoft.OpenApi" Version="1.6.15" />
Expand Down Expand Up @@ -52,5 +53,5 @@
<ItemGroup>
<Folder Include="Polyfill\" />
</ItemGroup>

</Project>
5 changes: 5 additions & 0 deletions src/fhir-codegen/Properties/launchSettings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,11 @@
"commandLineArgs": "compare -p hl7.fhir.r5.core#5.0.0 -c hl7.fhir.r4b.core#4.3.0 --auto-load-expansions --resolve-dependencies true --map-source-path ..\\..\\..\\fhir-cross-version --map-destination-path ..\\..\\..\\fhir-cross-version-source --map-save-style Source",
"workingDirectory": "$(MSBuildProjectDirectory)"
},
"Compare 50-60": {
"commandName": "Project",
"commandLineArgs": "compare -p hl7.fhir.r5.core#5.0.0 -c hl7.fhir.r6.core#6.0.0-ballot2 --auto-load-expansions --resolve-dependencies true --map-source-path ..\\..\\..\\fhir-cross-version --map-destination-path ..\\..\\..\\fhir-cross-version-source --map-save-style Source",
"workingDirectory": "$(MSBuildProjectDirectory)"
},
"Gui": {
"commandName": "Project",
"commandLineArgs": "gui",
Expand Down