wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added TestHarness and TraceLogHelper for assisting with automated simulation testing

This commit is contained in:
Evan Tschannen 2020-02-20 11:50:12 -08:00
parent 8f599e9d15
commit d7ab4db2b8
11 changed files with 2412 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1509
contrib/TestHarness/Program.cs Normal file
View File

File diff suppressed because it is too large Load Diff

56
contrib/TestHarness/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,56 @@
/*
* AssemblyInfo.cs
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("TestHarness")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Apple Inc.")]
[assembly: AssemblyProduct("TestHarness")]
[assembly: AssemblyCopyright("Copyright © Apple Inc. 2013")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("92d9df90-6d48-4558-bb25-d878ea69b4bd")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

62
contrib/TestHarness/TestHarness.csproj Normal file
View File

@ -0,0 +1,62 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
<ProductVersion>8.0.30703</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{6A6B7D20-EB7E-4768-BDE2-21FE0C32F17A}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>TestHarness</RootNamespace>
<AssemblyName>TestHarness</AssemblyName>
<TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
<TargetFrameworkProfile>Client</TargetFrameworkProfile>
<FileAlignment>512</FileAlignment>
<OutputPath>$(SolutionDir)bin\$(Configuration)\</OutputPath>
<IntermediateOutputPath>$(SystemDrive)\temp\msvcfdb\$(Configuration)\TestHarness\</IntermediateOutputPath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|AnyCPU'">
<DebugSymbols>true</DebugSymbols>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>AnyCPU</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisIgnoreBuiltInRuleSets>true</CodeAnalysisIgnoreBuiltInRuleSets>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|AnyCPU'">
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>AnyCPU</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisIgnoreBuiltInRules>true</CodeAnalysisIgnoreBuiltInRules>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\TraceLogHelper\TraceLogHelper.csproj">
<Project>{1FA45F13-1015-403C-9115-CEFFDD522B20}</Project>
<Name>TraceLogHelper</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

23
contrib/TestHarness/local.mk Normal file
View File

@ -0,0 +1,23 @@
#
# local.mk
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
TestHarness_LOCAL_REFERENCES := -lib:bin -r:TraceLogHelper.dll
bin/TestHarness.exe: bin/TraceLogHelper.dll

263
contrib/TraceLogHelper/Event.cs Normal file
View File

@ -0,0 +1,263 @@
/*
* Event.cs
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Dynamic;
namespace Magnesium
{
public enum Severity
{
SevDebug=5,
SevInfo=10,
SevWarn=20,
SevWarnAlways=30,
SevError=40,
};
public class Event
{
public double Time { get; set; }
public Severity Severity { get; set; }
public string Type { get; set; }
public string Machine { get; set; }
public string ID { get; set; }
public string WorkerDesc { get { return Machine + " " + ID; } }
public string TraceFile { get; set; }
public System.Xml.Linq.XElement original { get; set; }
static MyExpando emptyDetails = new MyExpando(new Dictionary<string, Object>());
MyExpando _Details = emptyDetails;
public dynamic Details { get{ return _Details; } }
public IDictionary<string,Object> DDetails { get { return _Details._members; }
set {
_Details = new MyExpando(value);
//foreach(var v in value)
// _Details.SetMember(v.Key, v.Value);
}
}
public Event ShallowCopy()
{
return (Event)MemberwiseClone();
}
class MyExpando : DynamicObject
{
public MyExpando(IDictionary<string, Object> init)
{
_members = init;
}
public IDictionary<string, Object> _members =
new Dictionary<string, Object>();
/// <summary>
/// When a new property is set,
/// add the property name and value to the dictionary
/// </summary>
public override bool TrySetMember
(SetMemberBinder binder, Object value)
{
if (!_members.ContainsKey(binder.Name))
_members.Add(binder.Name, value);
else
_members[binder.Name] = value;
return true;
}
public bool SetMember(string name, Object value)
{
if (!_members.ContainsKey(name))
_members.Add(name, value);
else
_members[name] = value;
return true;
}
/// <summary>
/// When user accesses something, return the value if we have it
/// </summary>
public override bool TryGetMember
(GetMemberBinder binder, out Object result)
{
if (_members.ContainsKey(binder.Name))
{
result = _members[binder.Name];
return true;
}
else
{
return base.TryGetMember(binder, out result);
}
}
/// <summary>
/// If a property value is a delegate, invoke it
/// </summary>
public override bool TryInvokeMember
(InvokeMemberBinder binder, Object[] args, out Object result)
{
if (_members.ContainsKey(binder.Name)
&& _members[binder.Name] is Delegate)
{
result = (_members[binder.Name] as Delegate).DynamicInvoke(args);
return true;
}
else
{
return base.TryInvokeMember(binder, args, out result);
}
}
/// <summary>
/// Return all dynamic member names
/// </summary>
/// <returns>
public override IEnumerable<string> GetDynamicMemberNames()
{
return _members.Keys;
}
}
public string FormatTestError(bool includeDetails)
{
string s = Type;
if (Type == "InternalError")
s = string.Format("{0} {1} {2}", Type, Details.File, Details.Line);
else if (Type == "TestFailure")
s = string.Format("{0} {1}", Type, Details.Reason);
else if (Type == "ValgrindError")
s = string.Format("{0} {1}", Type, Details.What);
else if (Type == "ExitCode")
s = string.Format("{0} 0x{1:x}", Type, int.Parse(Details.Code));
else if (Type == "StdErrOutput")
s = string.Format("{0}: {1}", Type, Details.Output);
else if (Type == "BTreeIntegrityCheck")
s = string.Format("{0}: {1}", Type, Details.ErrorDetail);
if (DDetails.ContainsKey("Error"))
s += " " + Details.Error;
if (DDetails.ContainsKey("WinErrorCode"))
s += " " + Details.WinErrorCode;
if (DDetails.ContainsKey("LinuxErrorCode"))
s += " " + Details.LinuxErrorCode;
if (DDetails.ContainsKey("Status"))
s += " Status=" + Details.Status;
if (DDetails.ContainsKey("In"))
s += " In " + Details.In;
if (DDetails.ContainsKey("SQLiteError"))
s += string.Format(" SQLiteError={0}({1})", Details.SQLiteError, Details.SQLiteErrorCode);
if (DDetails.ContainsKey("Details") && includeDetails)
s += ": " + Details.Details;
return s;
}
};
public class TestPlan : Event
{
public string TestUID;
public string TestFile;
public int randomSeed;
public bool Buggify;
public bool DeterminismCheck;
public string OldBinary;
};
public class Test : TestPlan
{
public string SourceVersion;
public double SimElapsedTime;
public double RealElapsedTime;
public bool ok;
public int passed, failed;
public int randomUnseed;
public long peakMemUsage;
public Event[] events; // Summarized events during the test
};
public struct AreaGraphPoint
{
public double X { get; set; }
public double Y { get; set; }
};
public struct LineGraphPoint
{
public double X { get; set; }
public double Y { get; set; }
public object Category { get; set; }
};
public class Interval
{
public double Begin { get; set; }
public double End { get; set; }
public string Category { get; set; }
public string Color { get; set; }
public string Detail { get; set; }
public object Object { get; set; }
};
public class MachineRole : Interval
{
public string Machine { get; set; }
public string Role { get; set; }
};
public class Location
{
public string Name;
public int Y;
};
//Specifies the type of LocationTime being used
public enum LocationTimeOp
{
//Designates a location in the code at a particular time
Normal = 0,
//Designates that this LocationTime maps one event ID to another
MapId
};
//Struct which houses a location in the code and a time which that location was hit
//The ID signifies the particular pass through the code
//Some of these objects act as special markers used to connect one ID to another
public struct LocationTime
{
public int id;
public double Time;
public Location Loc;
public LocationTimeOp locationTimeOp;
//If locationTimeOp == MapId, then this will hold the id of the event that should come after the id specified in the id field
public int childId;
};
}

93
contrib/TraceLogHelper/JsonParser.cs Normal file
View File

@ -0,0 +1,93 @@
/*
* JsonParser.cs
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Json;
using System.Text;
using System.Xml;
using System.Xml.XPath;
using System.Xml.Linq;
namespace Magnesium
{
public static class JsonParser
{
static Random r = new Random();
public static IEnumerable<Event> Parse(System.IO.Stream stream, string file,
bool keepOriginalElement = false, double startTime = -1, double endTime = Double.MaxValue,
double samplingFactor = 1.0)
{
using (var reader = new System.IO.StreamReader(stream))
{
string line;
while((line = reader.ReadLine()) != null)
{
XElement root = XElement.Load(JsonReaderWriterFactory.CreateJsonReader(new MemoryStream(Encoding.UTF8.GetBytes(line)), new XmlDictionaryReaderQuotas()));
Event ev = null;
try
{
ev = ParseEvent(root, file, keepOriginalElement, startTime, endTime, samplingFactor);
}
catch (Exception e)
{
throw new Exception(string.Format("Failed to parse {0}", root), e);
}
if (ev != null) yield return ev;
}
}
}
private static Event ParseEvent(XElement xEvent, string file, bool keepOriginalElement, double startTime, double endTime, double samplingFactor)
{
if (samplingFactor != 1.0 && r.NextDouble() > samplingFactor)
return null;
XElement trackLatestElement = xEvent.XPathSelectElement("//TrackLatestType");
bool rolledEvent = trackLatestElement != null && trackLatestElement.Value.Equals("Rolled");
String timeElement = (rolledEvent) ? "OriginalTime" : "Time";
double eventTime = double.Parse(xEvent.XPathSelectElement("//" + timeElement).Value);
if (eventTime < startTime || eventTime > endTime)
return null;
return new Event {
Severity = (Severity)int.Parse(xEvent.XPathSelectElement("//Severity").ValueOrDefault("40")),
Type = string.Intern(xEvent.XPathSelectElement("//Type").Value),
Time = eventTime,
Machine = string.Intern(xEvent.XPathSelectElement("//Machine").Value),
ID = string.Intern(xEvent.XPathSelectElement("//ID").ValueOrDefault("0")),
TraceFile = file,
DDetails = xEvent.Elements()
.Where(a=>a.Name != "Type" && a.Name != "Time" && a.Name != "Machine" && a.Name != "ID" && a.Name != "Severity" && (!rolledEvent || a.Name != "OriginalTime"))
.ToDictionary(a=>string.Intern(a.Name.LocalName), a=>(object)a.Value),
original = keepOriginalElement ? xEvent : null,
};
}
private static string ValueOrDefault( this XElement attr, string def ) {
if (attr == null) return def;
else return attr.Value;
}
}
}

56
contrib/TraceLogHelper/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,56 @@
/*
* AssemblyInfo.cs
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("TraceLogHelper")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Apple Inc.")]
[assembly: AssemblyProduct("TraceLogHelper")]
[assembly: AssemblyCopyright("Copyright © Apple Inc. 2013")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("ec8f2fdc-25fe-4958-b807-c30bd1da341b")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

58
contrib/TraceLogHelper/TraceLogHelper.csproj Normal file
View File

@ -0,0 +1,58 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProductVersion>8.0.30703</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{1FA45F13-1015-403C-9115-CEFFDD522B20}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>TraceLogHelper</RootNamespace>
<AssemblyName>TraceLogHelper</AssemblyName>
<TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>..\bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>..\bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Event.cs" />
<Compile Include="JsonParser.cs" />
<Compile Include="TraceLogUtil.cs" />
<Compile Include="XmlParser.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

79
contrib/TraceLogHelper/TraceLogUtil.cs Normal file
View File

@ -0,0 +1,79 @@
/*
* TraceLogUtil.cs
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Magnesium
{
public static class TraceLogUtil
{
public static IEnumerable<Event> IdentifyFailedTestPlans(IEnumerable<Event> events)
{
var failedPlans = new Dictionary<string, TestPlan>();
foreach (var ev in events)
{
var tp = ev as TestPlan;
if (tp == null || tp.TestUID == "")
{
yield return ev;
continue;
}
var t = tp as Test;
if (t == null)
{
if (!failedPlans.ContainsKey(tp.TestUID + tp.TraceFile))
{
failedPlans.Add(tp.TestUID + tp.TraceFile, tp);
}
}
else
{
failedPlans.Remove(tp.TestUID + tp.TraceFile);
if ((tp.TraceFile != null) && tp.TraceFile.EndsWith("-2.txt")) failedPlans.Remove(tp.TestUID + tp.TraceFile.Split('-')[0] + "-1.txt");
yield return ev;
}
}
foreach (var p in failedPlans.Values)
yield return new Test
{
Type = "FailedTestPlan",
Time = p.Time,
Machine = p.Machine,
TestUID = p.TestUID,
TestFile = p.TestFile,
randomSeed = p.randomSeed,
Buggify = p.Buggify,
DeterminismCheck = p.DeterminismCheck,
OldBinary = p.OldBinary,
events = new Event[] {
new Event {
Severity = Severity.SevWarnAlways,
Type = "TestNotSummarized",
Time = p.Time,
Machine = p.Machine
}
}
};
}
}
}

194
contrib/TraceLogHelper/XmlParser.cs Normal file
View File

@ -0,0 +1,194 @@
/*
* XmlParser.cs
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using System.Xml.Linq;
namespace Magnesium
{
public static class XmlParser
{
static Random r = new Random();
public static IEnumerable<Event> Parse(System.IO.Stream stream, string file,
bool keepOriginalElement = false, double startTime = -1, double endTime = Double.MaxValue,
double samplingFactor = 1.0)
{
using (var reader = XmlReader.Create(stream))
{
reader.ReadToDescendant("Trace");
reader.Read();
foreach (var xev in StreamElements(reader))
{
Event ev = null;
try
{
if (xev.Name == "Event")
ev = ParseEvent(xev, file, keepOriginalElement, startTime, endTime, samplingFactor);
else if (xev.Name == "Test")
ev = ParseTest(xev, file, keepOriginalElement);
else if (xev.Name == "TestPlan")
ev = ParseTestPlan(xev, file, keepOriginalElement);
}
catch (Exception e)
{
throw new Exception(string.Format("Failed to parse {0}", xev), e);
}
if (ev != null) yield return ev;
}
}
}
private static Event ParseEvent(XElement xEvent, string file, bool keepOriginalElement, double startTime, double endTime, double samplingFactor)
{
if (samplingFactor != 1.0 && r.NextDouble() > samplingFactor)
return null;
XAttribute trackLatestAttribute = xEvent.Attribute("TrackLatestType");
bool rolledEvent = trackLatestAttribute != null && trackLatestAttribute.Value.Equals("Rolled");
String timeAttribute = (rolledEvent) ? "OriginalTime" : "Time";
double eventTime = double.Parse(xEvent.Attribute(timeAttribute).Value);
if (eventTime < startTime || eventTime > endTime)
return null;
return new Event {
Severity = (Severity)int.Parse(xEvent.Attribute("Severity").ValueOrDefault("40")),
Type = string.Intern(xEvent.Attribute("Type").Value),
Time = eventTime,
Machine = string.Intern(xEvent.Attribute("Machine").Value),
ID = string.Intern(xEvent.Attribute("ID").ValueOrDefault("0")),
TraceFile = file,
DDetails = xEvent.Attributes()
.Where(a=>a.Name != "Type" && a.Name != "Time" && a.Name != "Machine" && a.Name != "ID" && a.Name != "Severity" && (!rolledEvent || a.Name != "OriginalTime"))
.ToDictionary(a=>string.Intern(a.Name.LocalName), a=>(object)a.Value),
original = keepOriginalElement ? xEvent : null,
};
}
private static string ValueOrDefault( this XAttribute attr, string def ) {
if (attr == null) return def;
else return attr.Value;
}
private static TestPlan ParseTestPlan(XElement xTP, string file, bool keepOriginalElement)
{
var time = double.Parse(xTP.Attribute("Time").ValueOrDefault("0"));
var machine = xTP.Attribute("Machine").ValueOrDefault("");
return new TestPlan
{
TraceFile = file,
Type = "TestPlan",
Time = time,
Machine = machine,
TestUID = xTP.Attribute("TestUID").ValueOrDefault(""),
TestFile = xTP.Attribute("TestFile").ValueOrDefault(""),
randomSeed = int.Parse(xTP.Attribute("RandomSeed").ValueOrDefault("0")),
Buggify = xTP.Attribute("BuggifyEnabled").ValueOrDefault("1") != "0",
DeterminismCheck = xTP.Attribute("DeterminismCheck").ValueOrDefault("1") != "0",
OldBinary = xTP.Attribute("OldBinary").ValueOrDefault(""),
original = keepOriginalElement ? xTP : null,
};
}
private static Test ParseTest(XElement xTest, string file, bool keepOriginalElement)
{
var time = double.Parse(xTest.Attribute("Time").ValueOrDefault("0"));
var machine = xTest.Attribute("Machine").ValueOrDefault("");
return new Test
{
TraceFile = file,
Type = "Test",
Time = time,
Machine = machine,
TestUID = xTest.Attribute("TestUID").ValueOrDefault(""),
TestFile = xTest.Attribute("TestFile").ValueOrDefault(""),
SourceVersion = xTest.Attribute("SourceVersion").ValueOrDefault(""),
ok = bool.Parse(xTest.Attribute("OK").ValueOrDefault("false")),
randomSeed = int.Parse(xTest.Attribute("RandomSeed").ValueOrDefault("0")),
randomUnseed = int.Parse(xTest.Attribute("RandomUnseed").ValueOrDefault("0")),
SimElapsedTime = double.Parse(xTest.Attribute("SimElapsedTime").ValueOrDefault("0")),
RealElapsedTime = double.Parse(xTest.Attribute("RealElapsedTime").ValueOrDefault("0")),
passed = int.Parse(xTest.Attribute("Passed").ValueOrDefault("0")),
failed = int.Parse(xTest.Attribute("Failed").ValueOrDefault("0")),
peakMemUsage = long.Parse(xTest.Attribute("PeakMemory").ValueOrDefault("0")),
Buggify = xTest.Attribute("BuggifyEnabled").ValueOrDefault("1") != "0",
DeterminismCheck = xTest.Attribute("DeterminismCheck").ValueOrDefault("1") != "0",
OldBinary = xTest.Attribute("OldBinary").ValueOrDefault(""),
original = keepOriginalElement ? xTest : null,
events = xTest.Elements().Select(e =>
new Event {
Severity = (Severity)int.Parse(e.Attribute("Severity").ValueOrDefault("0")),
Type = e.Name.LocalName,
Time = time,
Machine = machine,
DDetails = e.Attributes()
.Where(a => a.Name != "Type" && a.Name != "Time" && a.Name != "Machine" && a.Name != "Severity")
.ToDictionary(a => a.Name.LocalName, a => (object)a.Value)
}).ToArray()
};
}
private static T Try<T>(Func<T> action, Func<Exception, T> onError, Func<bool> isEOF)
{
try
{
return action();
}
catch (Exception e)
{
if (isEOF())
return onError(e);
else
throw e;
}
}
private static IEnumerable<XElement> StreamElements(this XmlReader reader)
{
while (!reader.EOF)
{
if (reader.NodeType == XmlNodeType.Element)
{
XElement node = null;
try
{
node = XElement.ReadFrom(reader) as XElement;
}
catch (Exception) { break; }
if (node != null)
yield return node;
}
else
{
try
{
reader.Read();
}
catch (Exception) { break; }
}
}
}
}
}

19
contrib/TraceLogHelper/local.mk Normal file
View File

@ -0,0 +1,19 @@
#
# local.mk
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#