Skip to content

Commit

Permalink
[MERGE #4516 @Cellule] Bytecode header tests + normalization
Browse files Browse the repository at this point in the history 
Merge pull request #4516 from Cellule:bytecodeheader_test

Add unit test to regenerate bytecode header to verify that what the current build generates matches with the current header file.
Normalize line endings before serializing bytecode header for library code.

ChakraFull PR: https://devdiv.visualstudio.com/DevDiv/_git/Chakra/pullrequest/100648?_a=overview
  • Loading branch information
@Cellule
Cellule committed Jan 10, 2018
2 parents d91fb1d + ed00225 commit f6f84bb
Show file tree
Hide file tree
Showing 12 changed files with 11,488 additions and 11,354 deletions.
93 changes: 47 additions & 46 deletions bin/ch/ch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,51 +181,59 @@ HRESULT CreateLibraryByteCodeHeader(LPCSTR contentsRaw, JsFinalizeCallback conte
DWORD written;
// For validating the header file against the library file
auto outputStr =
"//-------------------------------------------------------------------------------------------------------\r\n"
"// Copyright (C) Microsoft. All rights reserved.\r\n"
"// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.\r\n"
"//-------------------------------------------------------------------------------------------------------\r\n"
"#if 0\r\n";

"//-------------------------------------------------------------------------------------------------------\n"
"// Copyright (C) Microsoft. All rights reserved.\n"
"// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.\n"
"//-------------------------------------------------------------------------------------------------------\n"
"#if 0\n";

std::string normalizedContentStr;
char* nextToken = nullptr;
char* token = strtok_s((char*)contentsRaw, "\r", &nextToken);
while (token)
{
normalizedContentStr.append(token);
token = strtok_s(nullptr, "\r", &nextToken);
}
// We no longer need contentsRaw, so call the finalizer for it if one was provided
if (contentsRawFinalizeCallback != nullptr)
{
contentsRawFinalizeCallback((void*)contentsRaw);
}

const char* normalizedContent = normalizedContentStr.c_str();
// We still need contentsRaw after this, so pass a null finalizeCallback into it
HRESULT hr = GetSerializedBuffer(contentsRaw, nullptr, &bufferVal);
HRESULT hr = GetSerializedBuffer(normalizedContent, nullptr, &bufferVal);

IfFailedGoLabel((hr), ErrorRunFinalize);

IfJsrtErrorHRLabel(ChakraRTInterface::JsGetArrayBufferStorage(bufferVal, &bcBuffer, &bcBufferSize), ErrorRunFinalize);

bcFileHandle = CreateFile(bcFullPath, GENERIC_WRITE, FILE_SHARE_DELETE,
nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
if (bcFileHandle == INVALID_HANDLE_VALUE)
if (bcFullPath)
{
IfFailedGoLabel(E_FAIL, ErrorRunFinalize);
}

IfFalseGoLabel(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr), ErrorRunFinalize);
IfFalseGoLabel(WriteFile(bcFileHandle, contentsRaw, lengthBytes, &written, nullptr), ErrorRunFinalize);
if (lengthBytes < 2 || contentsRaw[lengthBytes - 2] != '\r' || contentsRaw[lengthBytes - 1] != '\n')
{
outputStr = "\r\n#endif\r\n";
bcFileHandle = CreateFile(bcFullPath, GENERIC_WRITE, FILE_SHARE_DELETE,
nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
}
else
{
outputStr = "#endif\r\n";
bcFileHandle = GetStdHandle(STD_OUTPUT_HANDLE);
}

// We no longer need contentsRaw, so call the finalizer for it if one was provided
if(contentsRawFinalizeCallback != nullptr)
if (bcFileHandle == INVALID_HANDLE_VALUE)
{
contentsRawFinalizeCallback((void*)contentsRaw);
IfFailedGoLabel(E_FAIL, ErrorRunFinalize);
}

IfFalseGoLabel(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr), ErrorRunFinalize);
IfFalseGoLabel(WriteFile(bcFileHandle, normalizedContent, (DWORD)normalizedContentStr.size(), &written, nullptr), ErrorRunFinalize);
outputStr = "\n#endif\n";

IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));

// Write out the bytecode
outputStr = "namespace Js\r\n{\r\n const char Library_Bytecode_";
outputStr = "namespace Js\n{\n const char Library_Bytecode_";
IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));
IfFalseGo(WriteFile(bcFileHandle, libraryNameNarrow, (DWORD)strlen(libraryNameNarrow), &written, nullptr));
outputStr = "[] = {\r\n/* 00000000 */";
outputStr = "[] = {\n/* 00000000 */";
IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));

for (unsigned int i = 0; i < bcBufferSize; i++)
Expand All @@ -249,24 +257,17 @@ HRESULT CreateLibraryByteCodeHeader(LPCSTR contentsRaw, JsFinalizeCallback conte
if (i % 16 == 15 && i < bcBufferSize - 1)
{
char offset[17];
_snprintf_s(offset, sizeof(offset), _countof(offset), "\r\n/* %08X */", i + 1); // close quote, new line, offset and open quote
IfFalseGo(WriteFile(bcFileHandle, offset, (DWORD)strlen(offset), &written, nullptr));
int actualLen = _snprintf_s(offset, sizeof(offset), _countof(offset), "\n/* %08X */", i + 1); // close quote, new line, offset and open quote
IfFalseGo(WriteFile(bcFileHandle, offset, actualLen, &written, nullptr));
}
}
outputStr = "};\r\n\r\n";
outputStr = "};\n\n";
IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));

outputStr = "}\r\n";
outputStr = "}\n";
IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));

if(false)
{
ErrorRunFinalize:
if(contentsRawFinalizeCallback != nullptr)
{
contentsRawFinalizeCallback((void*)contentsRaw);
}
}
Error:
if (bcFileHandle != nullptr)
{
Expand Down Expand Up @@ -716,19 +717,19 @@ HRESULT ExecuteTest(const char* fileName)
len = strlen(fullPath);
if (HostConfigFlags::flags.GenerateLibraryByteCodeHeaderIsEnabled)
{
if (HostConfigFlags::flags.GenerateLibraryByteCodeHeader != nullptr && *HostConfigFlags::flags.GenerateLibraryByteCodeHeader != _u('\0'))
{
CHAR libraryName[_MAX_PATH];
CHAR ext[_MAX_EXT];
_splitpath_s(fullPath, NULL, 0, NULL, 0, libraryName, _countof(libraryName), ext, _countof(ext));

IfFailGo(CreateLibraryByteCodeHeader(fileContents, WScriptJsrt::FinalizeFree, lengthBytes, HostConfigFlags::flags.GenerateLibraryByteCodeHeader, libraryName));
}
else
if (HostConfigFlags::flags.GenerateLibraryByteCodeHeader != nullptr)
{
fwprintf(stderr, _u("FATAL ERROR: -GenerateLibraryByteCodeHeader must provide the file name, i.e., -GenerateLibraryByteCodeHeader:<bytecode file name>, exiting\n"));
IfFailGo(E_FAIL);
if (wcslen(HostConfigFlags::flags.GenerateLibraryByteCodeHeader) == 0)
{
HostConfigFlags::flags.GenerateLibraryByteCodeHeader = nullptr;
}
}
CHAR libraryName[_MAX_PATH];
CHAR ext[_MAX_EXT];
_splitpath_s(fullPath, NULL, 0, NULL, 0, libraryName, _countof(libraryName), ext, _countof(ext));

IfFailGo(CreateLibraryByteCodeHeader(fileContents, WScriptJsrt::FinalizeFree, lengthBytes, HostConfigFlags::flags.GenerateLibraryByteCodeHeader, libraryName));
}
else if (HostConfigFlags::flags.SerializedIsEnabled)
{
Expand Down
92 changes: 90 additions & 2 deletions bin/rl/rl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,8 +274,10 @@ const char * const TestInfoKindName[] =
"command",
"timeout",
"sourcepath",
"eol-normalization",
NULL
};
static_assert((sizeof(TestInfoKindName) / sizeof(TestInfoKindName[0])) - 1 == TestInfoKind::_TIK_COUNT, "Fix the buffer size");

const char * const DirectiveNames[] =
{
Expand Down Expand Up @@ -872,14 +874,64 @@ HANDLE OpenFileToCompare(char *file)
return h;
}

struct MyFileWithoutCarriageReturn
{
CHandle* handle;
char* buf = nullptr;
size_t i = 0;
DWORD count = 0;
MyFileWithoutCarriageReturn(CHandle* handle, char* buf) : handle(handle), buf(buf) { Read(); }
bool readingError;
void Read()
{
i = 0;
readingError = !ReadFile(*handle, buf, CMPBUF_SIZE, &count, NULL);
}
bool HasNextChar()
{
if (count == 0)
{
return false;
}
if (i == count)
{
Read();
if (readingError)
{
return false;
}
return HasNextChar();
}
while (buf[i] == '\r')
{
++i;
if (i == count)
{
Read();
if (readingError)
{
return false;
}
return HasNextChar();
}
}
return true;
}
char GetNextChar()
{
return buf[i++];
}
};

// Do a quick file equality comparison using pure Win32 functions. (Avoid
// using CRT functions; the MT CRT seems to have locking/flushing problems on
// MP boxes.)

int
DoCompare(
char *file1,
char *file2
char *file2,
BOOL normalizeLineEndings
)
{
CHandle h1, h2; // automatically closes open handles
Expand All @@ -906,7 +958,42 @@ DoCompare(
return -1;
}

// Short circuit by first checking for different file lengths.
if (normalizeLineEndings)
{
MyFileWithoutCarriageReturn f1(&h1, cmpbuf1);
MyFileWithoutCarriageReturn f2(&h2, cmpbuf2);
if (f1.readingError || f2.readingError)
{
LogError("ReadFile failed doing compare of %s and %s", file1, file2);
return -1;
}
while (f1.HasNextChar() && f2.HasNextChar())
{
if (f1.readingError || f2.readingError)
{
LogError("ReadFile failed doing compare of %s and %s", file1, file2);
return -1;
}

if (f1.GetNextChar() != f2.GetNextChar())
{
#ifndef NODEBUG
if (FDebug)
printf("DoCompare shows %s and %s are NOT equal (contents differ)\n", file1, file2);
#endif
return 1;
}
}
if (f1.HasNextChar() != f2.HasNextChar())
{
#ifndef NODEBUG
if (FDebug)
printf("DoCompare shows %s and %s are NOT equal (contents differ)\n", file1, file2);
#endif
return 1;
}
return 0;
}

size1 = GetFileSize(h1, NULL); // assume < 4GB files
if (size1 == 0xFFFFFFFF) {
Expand Down Expand Up @@ -2378,6 +2465,7 @@ WriteEnvLst
NULL,
NULL,
NULL,
NULL,
NULL
};

Expand Down
3 changes: 2 additions & 1 deletion bin/rl/rl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ enum TestInfoKind
TIK_COMMAND,
TIK_TIMEOUT,
TIK_SOURCE_PATH,
TIK_EOL_NORMALIZATION,
_TIK_COUNT
};

Expand Down Expand Up @@ -920,7 +921,7 @@ extern void __cdecl LogOut(const char *fmt, ...);
extern void __cdecl LogError(const char *fmt, ...);
extern void FlushOutput(void);
extern char *mytmpnam(const char* directory, const char *prefix, char *filename);
extern int DoCompare(char *file1, char *file2);
extern int DoCompare(char *file1, char *file2, BOOL normalizeLineEndings = false);
extern void UpdateTitleStatus();
extern int mystrcmp(const char *a, const char *b);
extern char * mystrtok(char *s, const char *delim, const char *term);
Expand Down
4 changes: 2 additions & 2 deletions bin/rl/rlregr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ RegrFile(
return -1;
}

x = DoCompare(fullasmbuf, fullmasterasmbuf);
x = DoCompare(fullasmbuf, fullmasterasmbuf, pTestVariant->testInfo.hasData[TIK_EOL_NORMALIZATION]);
if (x < 0)
return -1;

Expand Down Expand Up @@ -378,7 +378,7 @@ RegrFile(

// Compare again.

x = DoCompare(tmp_file1, tmp_file2);
x = DoCompare(tmp_file1, tmp_file2, pTestVariant->testInfo.hasData[TIK_EOL_NORMALIZATION]);
if (x < 0) {
rc = -1;
}
Expand Down
4 changes: 2 additions & 2 deletions bin/rl/rlrun.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@ int

sprintf_s(baseline_file, "%s\\%s", pDir->GetFullPathFromSourceOrDirectory(),
pTestVariant->testInfo.data[TIK_BASELINE]);
if (DoCompare(baseline_file, full)) {
if (DoCompare(baseline_file, full, pTestVariant->testInfo.hasData[TIK_EOL_NORMALIZATION])) {
reason = "diffs from baseline";
sprintf_s(optReportBuf, "%s", baseline_file);
fFailed = TRUE;
Expand Down Expand Up @@ -1044,7 +1044,7 @@ BOOL
else {
sprintf_s(full, "%s\\%s", pDir->GetFullPathFromSourceOrDirectory(),
pTestVariant->testInfo.data[TIK_BASELINE]);
if (DoCompare(tmp_file1, full)) {
if (DoCompare(tmp_file1, full, pTestVariant->testInfo.hasData[TIK_EOL_NORMALIZATION])) {

// Output differs, run spiff to see if it's just minor
// floating point anomalies.
Expand Down
Loading

0 comments on commit f6f84bb

Please sign in to comment.