Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

If no other training data, don't add any keyterms #580

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+41 −260
Open

If no other training data, don't add any keyterms #580

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
262 changes: 21 additions & 241 deletions src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,27 @@ public async Task RunAsync_EnableKeyTerms()
Assert.That(src1Count, Is.EqualTo(14));
Assert.That(src2Count, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(1));
Assert.That(termCount, Is.EqualTo(144));
Assert.That(termCount, Is.EqualTo(3416));
});
}

[Test]
public async Task RunAsync_EnableKeyTermsNoTrainingData()
{
using TestEnvironment env = new();
ParallelCorpus corpus1 = env.DefaultParatextCorpus;
corpus1.SourceCorpora[0].TrainOnTextIds = new HashSet<string>();
corpus1.TargetCorpora[0].TrainOnTextIds = new HashSet<string>();

await env.RunBuildJobAsync(corpus1, useKeyTerms: true);

(int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync();
Assert.Multiple(() =>
{
Assert.That(src1Count, Is.EqualTo(0));
Assert.That(src2Count, Is.EqualTo(0));
Assert.That(trgCount, Is.EqualTo(0));
Assert.That(termCount, Is.EqualTo(0));
});
}

Expand Down Expand Up @@ -508,246 +528,6 @@ public async Task ParallelCorpusAsync()
});
}

[Test]
public async Task ParallelCorpusAsync_UseKeyTerms()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
{
new ParallelCorpus()
{
Id = "1",
SourceCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"LEV",
new() { }
}
},
PretranslateChapters = new()
{
{
"1CH",
new() { }
}
}
},
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
}
},
PretranslateChapters = new() { }
},
},
TargetCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
}
}
},
new()
{
Id = "_2",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
TrainOnChapters = new()
{
{
"MAT",
new() { 1 }
},
{
"MRK",
new() { }
},
{
"LEV",
new() { }
}
}
}
}
}
};
await env.RunBuildJobAsync(corpora, useKeyTerms: true);
string source = await env.GetSourceExtractAsync();
string target = await env.GetTargetExtractAsync();
Assert.Multiple(() =>
{
StringAssert.StartsWith(
@"Source one, chapter fourteen, verse fifty-five. Segment b.
Source one, chapter fourteen, verse fifty-six.
Source two, chapter one, verse one.
Source two, chapter one, verse two.
Source two, chapter one, verse three.
Source one, chapter one, verse four.
Source two, chapter one, verse five. Source two, chapter one, verse six.
Source one, chapter one, verse seven, eight, and nine. Source one, chapter one, verse ten.
Source two, chapter one, verse one.
",
source
);
StringAssert.StartsWith(
@"Target two, chapter fourteen, verse fifty-five.
Target two, chapter fourteen, verse fifty-six.
Target one, chapter one, verse one.
Target one, chapter one, verse two.
Target one, chapter one, verse three.

Target one, chapter one, verse five and six.
Target one, chapter one, verse seven and eight. Target one, chapter one, verse nine and ten.

",
target
);
StringAssert.Contains("Abraham", source);
StringAssert.Contains("Abraham", target);
StringAssert.DoesNotContain("Zedekiah", source);
StringAssert.DoesNotContain("Zedekiah", target);
});
JsonArray? pretranslations = await env.GetPretranslationsAsync();
Assert.That(pretranslations, Is.Not.Null);
Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToJsonString());
Assert.That(
pretranslations[2]!["translation"]!.ToString(),
Is.EqualTo("Source one, chapter twelve, verse one.")
);
}

[Test]
public async Task ParallelCorpusAsync_UseKeyTerms_TextIds()
{
using TestEnvironment env = new();
var corpora = new List<ParallelCorpus>()
{
new ParallelCorpus()
{
Id = "1",
SourceCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source1") },
TrainOnTextIds = ["MAT", "LEV"],
PretranslateTextIds = ["1CH"]
},
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-source2") },
TrainOnTextIds = ["MAT", "MRK"],
PretranslateTextIds = []
},
},
TargetCorpora = new List<MonolingualCorpus>()
{
new()
{
Id = "_1",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target1") },
TrainOnTextIds = ["MAT", "MRK"]
},
new()
{
Id = "_2",
Language = "en",
Files = new List<CorpusFile> { env.ParatextFile("pt-target2") },
TrainOnTextIds = ["MAT", "MRK", "LEV"]
}
}
}
};
await env.RunBuildJobAsync(corpora, useKeyTerms: true);
string source = await env.GetSourceExtractAsync();
string target = await env.GetTargetExtractAsync();
Assert.Multiple(() =>
{
StringAssert.StartsWith(
@"Source one, chapter fourteen, verse fifty-five. Segment b.
Source one, chapter fourteen, verse fifty-six.
Source two, chapter one, verse one.
Source two, chapter one, verse two.
Source two, chapter one, verse three.
Source one, chapter one, verse four.
Source two, chapter one, verse five. Source two, chapter one, verse six.
Source one, chapter one, verse seven, eight, and nine. Source one, chapter one, verse ten.
Source one, chapter two, verse one.
Source one, chapter two, verse two.

Source two, chapter one, verse one.
",
source
);
StringAssert.StartsWith(
@"Target two, chapter fourteen, verse fifty-five.
Target two, chapter fourteen, verse fifty-six.
Target one, chapter one, verse one.
Target one, chapter one, verse two.
Target one, chapter one, verse three.

Target one, chapter one, verse five and six.
Target one, chapter one, verse seven and eight. Target one, chapter one, verse nine and ten.
Target one, chapter two, verse one.

Target one, chapter two, verse three.

",
target
);
StringAssert.Contains("Abraham", source);
StringAssert.Contains("Abraham", target);
StringAssert.DoesNotContain("Zedekiah", source);
StringAssert.DoesNotContain("Zedekiah", target);
});
JsonArray? pretranslations = await env.GetPretranslationsAsync();
Assert.That(pretranslations, Is.Not.Null);
Assert.That(pretranslations!.Count, Is.EqualTo(7), pretranslations.ToJsonString());
Assert.That(
pretranslations[2]!["translation"]!.ToString(),
Is.EqualTo("Source one, chapter twelve, verse one.")
);
}

private class TestEnvironment : DisposableBase
{
private static readonly string TestDataPath = Path.Combine(
Expand Down
8 changes: 3 additions & 5 deletions src/ServiceToolkit/src/SIL.ServiceToolkit/Services/CorpusService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,14 @@ public IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> file
return corpora;
}

public IEnumerable<ITextCorpus> CreateTermCorpora(
IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
)
public IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> corpusFiles)
{
foreach ((CorpusFile file, Dictionary<string, HashSet<int>> chapters) in corpora)
foreach (CorpusFile file in corpusFiles)
{
switch (file.Format)
{
case FileFormat.Paratext:
yield return new ParatextBackupTermsCorpus(file.Location, ["PN"], chapters: chapters);
yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]);
break;
}
}
Expand Down
4 changes: 1 addition & 3 deletions src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ICorpusService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,5 @@
public interface ICorpusService
{
IEnumerable<ITextCorpus> CreateTextCorpora(IReadOnlyList<CorpusFile> files);
IEnumerable<ITextCorpus> CreateTermCorpora(
IReadOnlyList<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> corpora
);
IEnumerable<ITextCorpus> CreateTermCorpora(IReadOnlyList<CorpusFile> corpusFiles);
}
27 changes: 16 additions & 11 deletions src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ public async Task PreprocessAsync(
bool useKeyTerms = false
)
{
bool parallelTrainingDataPresent = false;
List<Row> keyTermTrainingData = new();
foreach (ParallelCorpus corpus in corpora)
{
(MonolingualCorpus Corpus, ITextCorpus TextCorpus)[] sourceCorpora = corpus
Expand Down Expand Up @@ -78,23 +80,19 @@ public async Task PreprocessAsync(
foreach (Row row in CollapseRanges(trainingRows))
{
await train(row);
if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0)
{
parallelTrainingDataPresent = true;
}
}

if (useKeyTerms)
{
ITextCorpus[]? sourceTermCorpora = _corpusService
.CreateTermCorpora(
sourceCorpora
.SelectMany(corpus => GetChaptersPerFile(corpus.Corpus, corpus.TextCorpus))
.ToArray()
)
.CreateTermCorpora(sourceCorpora.SelectMany(corpus => corpus.Corpus.Files).ToArray())
.ToArray();
ITextCorpus[]? targetTermCorpora = _corpusService
.CreateTermCorpora(
targetCorpora
.SelectMany(corpus => GetChaptersPerFile(corpus.Corpus, corpus.TextCorpus))
.ToArray()
)
.CreateTermCorpora(targetCorpora.SelectMany(corpus => corpus.Corpus.Files).ToArray())
.ToArray();
if (sourceTermCorpora is not null && targetTermCorpora is not null)
{
Expand All @@ -107,7 +105,7 @@ ParallelTextRow row in parallelKeyTermsCorpus.DistinctBy(row =>
)
)
{
await train(new Row(row.TextId, row.Refs, row.SourceText, row.TargetText, 1));
keyTermTrainingData.Add(new Row(row.TextId, row.Refs, row.SourceText, row.TargetText, 1));
}
}
}
Expand All @@ -123,6 +121,13 @@ ParallelTextRow row in parallelKeyTermsCorpus.DistinctBy(row =>
await pretranslate(row, corpus);
}
}
if (useKeyTerms && parallelTrainingDataPresent)
{
foreach (Row row in keyTermTrainingData)
{
await train(row);
}
}
}

private static IEnumerable<(CorpusFile File, Dictionary<string, HashSet<int>> Chapters)> GetChaptersPerFile(
Expand Down
Loading