From c5e8f83db1606a89e70a9f4a0ac2fc03dea2418c Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Tue, 25 Jun 2024 19:29:23 +0200 Subject: [PATCH] Tar: support GNU numeric format. (#101172) The tar specification stores numeric fields using an octal representation. This limits the range of values that can be stored. To increase the supported range, a GNU extension defines that when the leading byte is 0xff/0x80 the remaining bytes are a negative/positive big endian formatted value. When writing under the PAX format, we continue to only use the only octal representation in the header fields. The values are overridden using extended attributes. --- .../src/Resources/Strings.resx | 4 +- .../src/System/Formats/Tar/GnuTarEntry.cs | 2 - .../src/System/Formats/Tar/PosixTarEntry.cs | 14 +- .../src/System/Formats/Tar/TarEntry.cs | 10 +- .../src/System/Formats/Tar/TarHeader.Read.cs | 19 +- .../src/System/Formats/Tar/TarHeader.Write.cs | 123 ++++++++++--- .../src/System/Formats/Tar/TarHelpers.cs | 24 ++- .../tests/Manual/ManualTests.cs | 3 +- .../TarReader/TarReader.File.Async.Tests.cs | 11 +- .../tests/TarReader/TarReader.File.Tests.cs | 11 +- .../tests/TarTestsBase.Gnu.cs | 21 ++- .../tests/TarTestsBase.Posix.cs | 42 ++++- .../System.Formats.Tar/tests/TarTestsBase.cs | 12 +- .../TarWriter/TarWriter.WriteEntry.Base.cs | 75 ++++++++ .../TarWriter.WriteEntry.Entry.Pax.Tests.cs | 88 ++------- .../TarWriter/TarWriter.WriteEntry.Tests.cs | 167 ++++++++++------- ...rWriter.WriteEntryAsync.Entry.Pax.Tests.cs | 88 ++------- .../TarWriter.WriteEntryAsync.Tests.cs | 168 ++++++++++-------- 18 files changed, 539 insertions(+), 343 deletions(-) diff --git a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx index a7b4cf8d53a37..ac632535dfd18 100644 --- a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx +++ b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx @@ -193,8 +193,8 @@ The field '{0}' exceeds the maximum allowed length for this format. - - The value of the size field for the current entry of format '{0}' is greater than the format allows. + + The value of the field for the current entry of format '{0}' is greater than the format allows. The extended attribute key '{0}' contains a disallowed '{1}' character. diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs index e8acadd9fcf4b..ee3bad50c7450 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs @@ -98,7 +98,6 @@ public DateTimeOffset AccessTime get => _header._aTime; set { - ArgumentOutOfRangeException.ThrowIfLessThan(value, DateTimeOffset.UnixEpoch); _header._aTime = value; } } @@ -112,7 +111,6 @@ public DateTimeOffset ChangeTime get => _header._cTime; set { - ArgumentOutOfRangeException.ThrowIfLessThan(value, DateTimeOffset.UnixEpoch); _header._cTime = value; } } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs index 73d2f01b42335..72d1396f83bee 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs @@ -50,7 +50,7 @@ internal PosixTarEntry(TarEntry other, TarEntryFormat format) /// /// Character and block devices are Unix-specific entry types. /// The entry does not represent a block device or a character device. - /// The value is negative, or larger than 2097151. + /// The value is negative, or larger than 2097151 when using or . public int DeviceMajor { get => _header._devMajor; @@ -62,7 +62,10 @@ public int DeviceMajor } ArgumentOutOfRangeException.ThrowIfNegative(value); - ArgumentOutOfRangeException.ThrowIfGreaterThan(value, 0x1FFFFF); // 7777777 in octal + if (FormatIsOctalOnly) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(value, 0x1FFFFF); // 7777777 in octal + } _header._devMajor = value; } @@ -73,7 +76,7 @@ public int DeviceMajor /// /// Character and block devices are Unix-specific entry types. /// The entry does not represent a block device or a character device. - /// The value is negative, or larger than 2097151. + /// The value is negative, or larger than 2097151 when using or . public int DeviceMinor { get => _header._devMinor; @@ -85,7 +88,10 @@ public int DeviceMinor } ArgumentOutOfRangeException.ThrowIfNegative(value); - ArgumentOutOfRangeException.ThrowIfGreaterThan(value, 0x1FFFFF); // 7777777 in octal + if (FormatIsOctalOnly) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(value, 0x1FFFFF); // 7777777 in octal + } _header._devMinor = value; } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs index 83f915875d266..73b158b10957b 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs @@ -20,6 +20,9 @@ public abstract partial class TarEntry // Used to access the data section of this entry in an unseekable file private TarReader? _readerOfOrigin; + // These formats have a limited numeric range due to the octal number representation. + protected bool FormatIsOctalOnly => _header._format is TarEntryFormat.V7 or TarEntryFormat.Ustar; + // Constructor called when reading a TarEntry from a TarReader. internal TarEntry(TarHeader header, TarReader readerOfOrigin, TarEntryFormat format) { @@ -92,13 +95,16 @@ public int Gid /// A timestamps that represents the last time the contents of the file represented by this entry were modified. /// /// In Unix platforms, this timestamp is commonly known as mtime. - /// The specified value is larger than . + /// The specified value is larger than when using or . public DateTimeOffset ModificationTime { get => _header._mTime; set { - ArgumentOutOfRangeException.ThrowIfLessThan(value, DateTimeOffset.UnixEpoch); + if (FormatIsOctalOnly) + { + ArgumentOutOfRangeException.ThrowIfLessThan(value, DateTimeOffset.UnixEpoch); + } _header._mTime = value; } } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs index 5036a59334d85..4d924c61056f1 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs @@ -374,8 +374,7 @@ private async Task ProcessDataBlockAsync(Stream archiveStream, bool copyData, Ca return null; } - long size = (long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Size, FieldLengths.Size)); - Debug.Assert(size <= TarHelpers.MaxSizeLength, "size exceeded the max value possible with 11 octal digits. Actual size " + size); + long size = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Size, FieldLengths.Size)); if (size < 0) { throw new InvalidDataException(SR.Format(SR.TarSizeFieldNegative)); @@ -384,14 +383,14 @@ private async Task ProcessDataBlockAsync(Stream archiveStream, bool copyData, Ca // Continue with the rest of the fields that require no special checks TarHeader header = new(initialFormat, name: TarHelpers.GetTrimmedUtf8String(buffer.Slice(FieldLocations.Name, FieldLengths.Name)), - mode: (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)), - mTime: TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch((long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.MTime, FieldLengths.MTime))), + mode: TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)), + mTime: TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.MTime, FieldLengths.MTime))), typeFlag: (TarEntryType)buffer[FieldLocations.TypeFlag]) { _checksum = checksum, _size = size, - _uid = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Uid, FieldLengths.Uid)), - _gid = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)), + _uid = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Uid, FieldLengths.Uid)), + _gid = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)), _linkName = TarHelpers.GetTrimmedUtf8String(buffer.Slice(FieldLocations.LinkName, FieldLengths.LinkName)) }; @@ -524,10 +523,10 @@ private void ReadPosixAndGnuSharedAttributes(Span buffer) if (_typeFlag is TarEntryType.CharacterDevice or TarEntryType.BlockDevice) { // Major number for a character device or block device entry. - _devMajor = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.DevMajor, FieldLengths.DevMajor)); + _devMajor = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.DevMajor, FieldLengths.DevMajor)); // Minor number for a character device or block device entry. - _devMinor = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.DevMinor, FieldLengths.DevMinor)); + _devMinor = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.DevMinor, FieldLengths.DevMinor)); } } @@ -536,10 +535,10 @@ private void ReadPosixAndGnuSharedAttributes(Span buffer) private void ReadGnuAttributes(Span buffer) { // Convert byte arrays - long aTime = (long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.ATime, FieldLengths.ATime)); + long aTime = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.ATime, FieldLengths.ATime)); _aTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(aTime); - long cTime = (long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.CTime, FieldLengths.CTime)); + long cTime = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.CTime, FieldLengths.CTime)); _cTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(cTime); // TODO: Read the bytes of the currently unsupported GNU fields, in case user wants to write this entry into another GNU archive, they need to be preserved. https://github.com/dotnet/runtime/issues/68230 diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index a14c12443aea2..e0d003a657bc8 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Buffers.Binary; using System.Buffers.Text; using System.Collections.Generic; using System.Diagnostics; @@ -15,6 +16,9 @@ namespace System.Formats.Tar // Writes header attributes of a tar archive entry. internal sealed partial class TarHeader { + private const long Octal12ByteFieldMaxValue = (1L << (3 * 11)) - 1; // Max value of 11 octal digits. + private const int Octal8ByteFieldMaxValue = (1 << (3 * 7)) - 1; // Max value of 7 octal digits. + private static ReadOnlySpan UstarMagicBytes => "ustar\0"u8; private static ReadOnlySpan UstarVersionBytes => "00"u8; @@ -606,35 +610,22 @@ private int WriteCommonFields(Span buffer, TarEntryType actualEntryType) if (_mode > 0) { - checksum += FormatOctal(_mode, buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)); + checksum += FormatNumeric(_mode, buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)); } if (_uid > 0) { - checksum += FormatOctal(_uid, buffer.Slice(FieldLocations.Uid, FieldLengths.Uid)); + checksum += FormatNumeric(_uid, buffer.Slice(FieldLocations.Uid, FieldLengths.Uid)); } if (_gid > 0) { - checksum += FormatOctal(_gid, buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)); + checksum += FormatNumeric(_gid, buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)); } if (_size > 0) { - if (_size <= TarHelpers.MaxSizeLength) - { - checksum += FormatOctal(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size)); - } - else if (_format is not TarEntryFormat.Pax) - { - throw new ArgumentException(SR.Format(SR.TarSizeFieldTooLargeForEntryFormat, _format)); - } - else - { - // No writing, just verifications - Debug.Assert(_typeFlag is not TarEntryType.ExtendedAttributes and not TarEntryType.GlobalExtendedAttributes); - Debug.Assert(Convert.ToInt64(ExtendedAttributes[PaxEaSize]) > TarHelpers.MaxSizeLength); - } + checksum += FormatNumeric(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size)); } checksum += WriteAsTimestamp(_mTime, buffer.Slice(FieldLocations.MTime, FieldLengths.MTime)); @@ -739,12 +730,12 @@ private int WritePosixAndGnuSharedFields(Span buffer) if (_devMajor > 0) { - checksum += FormatOctal(_devMajor, buffer.Slice(FieldLocations.DevMajor, FieldLengths.DevMajor)); + checksum += FormatNumeric(_devMajor, buffer.Slice(FieldLocations.DevMajor, FieldLengths.DevMajor)); } if (_devMinor > 0) { - checksum += FormatOctal(_devMinor, buffer.Slice(FieldLocations.DevMinor, FieldLengths.DevMinor)); + checksum += FormatNumeric(_devMinor, buffer.Slice(FieldLocations.DevMinor, FieldLengths.DevMinor)); } return checksum; @@ -916,7 +907,7 @@ private void CollectExtendedAttributesFromStandardFieldsIfNeeded() ExtendedAttributes[PaxEaLinkName] = _linkName; } - if (_size > TarHelpers.MaxSizeLength) + if (_size > Octal12ByteFieldMaxValue) { ExtendedAttributes[PaxEaSize] = _size.ToString(); } @@ -925,6 +916,42 @@ private void CollectExtendedAttributesFromStandardFieldsIfNeeded() ExtendedAttributes.Remove(PaxEaSize); } + if (_uid > Octal8ByteFieldMaxValue) + { + ExtendedAttributes[PaxEaUid] = _uid.ToString(); + } + else + { + ExtendedAttributes.Remove(PaxEaUid); + } + + if (_gid > Octal8ByteFieldMaxValue) + { + ExtendedAttributes[PaxEaGid] = _gid.ToString(); + } + else + { + ExtendedAttributes.Remove(PaxEaGid); + } + + if (_devMajor > Octal8ByteFieldMaxValue) + { + ExtendedAttributes[PaxEaDevMajor] = _devMajor.ToString(); + } + else + { + ExtendedAttributes.Remove(PaxEaDevMajor); + } + + if (_devMinor > Octal8ByteFieldMaxValue) + { + ExtendedAttributes[PaxEaDevMinor] = _devMinor.ToString(); + } + else + { + ExtendedAttributes.Remove(PaxEaDevMinor); + } + // Sets the specified string to the dictionary if it's longer than the specified max byte length; otherwise, remove it. static void TryAddStringField(Dictionary extendedAttributes, string key, string? value, int maxLength) { @@ -1022,6 +1049,56 @@ private static int Checksum(ReadOnlySpan bytes) return checksum; } + private int FormatNumeric(int value, Span destination) + { + Debug.Assert(destination.Length == 8, "8 byte field expected."); + + bool isOctalRange = value >= 0 && value <= Octal8ByteFieldMaxValue; + + if (isOctalRange || _format == TarEntryFormat.Pax) + { + return FormatOctal(value, destination); + } + else if (_format == TarEntryFormat.Gnu) + { + // GNU format: store negative numbers in big endian format with leading '0xff' byte. + // store positive numbers in big endian format with leading '0x80' byte. + long destinationValue = value; + destinationValue |= 1L << 63; + BinaryPrimitives.WriteInt64BigEndian(destination, destinationValue); + return Checksum(destination); + } + else + { + throw new ArgumentException(SR.Format(SR.TarFieldTooLargeForEntryFormat, _format)); + } + } + + private int FormatNumeric(long value, Span destination) + { + Debug.Assert(destination.Length == 12, "12 byte field expected."); + const int Offset = 4; // 4 bytes before the long. + + bool isOctalRange = value >= 0 && value <= Octal12ByteFieldMaxValue; + + if (isOctalRange || _format == TarEntryFormat.Pax) + { + return FormatOctal(value, destination); + } + else if (_format == TarEntryFormat.Gnu) + { + // GNU format: store negative numbers in big endian format with leading '0xff' byte. + // store positive numbers in big endian format with leading '0x80' byte. + BinaryPrimitives.WriteUInt32BigEndian(destination, value < 0 ? 0xffffffff : 0x80000000); + BinaryPrimitives.WriteInt64BigEndian(destination.Slice(Offset), value); + return Checksum(destination); + } + else + { + throw new ArgumentException(SR.Format(SR.TarFieldTooLargeForEntryFormat, _format)); + } + } + // Writes the specified decimal number as a right-aligned octal number and returns its checksum. private static int FormatOctal(long value, Span destination) { @@ -1040,11 +1117,11 @@ private static int FormatOctal(long value, Span destination) return WriteRightAlignedBytesAndGetChecksum(digits.Slice(i), destination); } - // Writes the specified DateTimeOffset's Unix time seconds as a right-aligned octal number, and returns its checksum. - private static int WriteAsTimestamp(DateTimeOffset timestamp, Span destination) + // Writes the specified DateTimeOffset's Unix time seconds, and returns its checksum. + private int WriteAsTimestamp(DateTimeOffset timestamp, Span destination) { long unixTimeSeconds = timestamp.ToUnixTimeSeconds(); - return FormatOctal(unixTimeSeconds, destination); + return FormatNumeric(unixTimeSeconds, destination); } // Writes the specified text as an UTF8 string aligned to the left, and returns its checksum. diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs index 2d8f56a62eeb5..639ad480f18a6 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs @@ -19,7 +19,6 @@ internal static partial class TarHelpers { internal const short RecordSize = 512; internal const int MaxBufferLength = 4096; - internal const long MaxSizeLength = (1L << 33) - 1; // Max value of 11 octal digits = 2^33 - 1 or 8 Gb. internal const UnixFileMode ValidUnixFileModes = UnixFileMode.UserRead | @@ -215,6 +214,29 @@ internal static TarEntryType GetCorrectTypeFlagForFormat(TarEntryFormat format, return entryType; } + /// Parses a numeric field. + internal static T ParseNumeric(ReadOnlySpan buffer) where T : struct, INumber, IBinaryInteger + { + // The tar standard specifies that numeric fields are stored using an octal representation. + // This limits the range of values that can be stored in the fields. + // To increase the supported range, a GNU extension defines that when the leading byte is + // '0xff'/'0x80' the remaining bytes are a negative/positive big formatted endian value. + // Like the 'tar' tool we are permissive when encountering this representation in non GNU formats. + byte leadingByte = buffer[0]; + if (leadingByte == 0xff) + { + return T.ReadBigEndian(buffer, isUnsigned: false); + } + else if (leadingByte == 0x80) + { + return T.ReadBigEndian(buffer.Slice(1), isUnsigned: true); + } + else + { + return ParseOctal(buffer); + } + } + /// Parses a byte span that represents an ASCII string containing a number in octal base. internal static T ParseOctal(ReadOnlySpan buffer) where T : struct, INumber { diff --git a/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs b/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs index 1fa1c686e40e8..34cc32183170f 100644 --- a/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs +++ b/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs @@ -22,8 +22,9 @@ public static IEnumerable WriteEntry_LongFileSize_TheoryData() yield return new object[] { entryFormat, LegacyMaxFileSize, unseekableStream }; } - // Pax supports unlimited size files. + // Pax and Gnu supports unlimited size files. yield return new object[] { TarEntryFormat.Pax, LegacyMaxFileSize + 1, unseekableStream }; + yield return new object[] { TarEntryFormat.Gnu, LegacyMaxFileSize + 1, unseekableStream }; } } diff --git a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs index d86cfa4e34dd4..e603e6e9aa469 100644 --- a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs @@ -263,7 +263,6 @@ public async Task AllowSpacesInOctalFieldsAsync(string folderName, string testCa [InlineData("invalid-go17")] // Many octal fields are all zero chars [InlineData("issue11169")] // Checksum with null in the middle [InlineData("issue10968")] // Garbage chars - [InlineData("writer-big")] // The size field contains an euro char public async Task Throw_ArchivesWithRandomCharsAsync(string testCaseName) { await using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", testCaseName); @@ -271,6 +270,16 @@ public async Task Throw_ArchivesWithRandomCharsAsync(string testCaseName) await Assert.ThrowsAsync(async () => await reader.GetNextEntryAsync()); } + [Fact] + public async Task Throw_ArchiveIsShortAsync() + { + // writer-big has a header for a 16G file but not its contents. + using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", "writer-big"); + using TarReader reader = new TarReader(archiveStream); + // MemoryStream throws when we try to change its Position past its Length. + await Assert.ThrowsAsync(async () => await reader.GetNextEntryAsync()); + } + [Fact] public async Task GarbageEntryChecksumZeroReturnNullAsync() { diff --git a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs index 6b5dae94caf6b..955f29c841849 100644 --- a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs @@ -271,7 +271,6 @@ public void AllowSpacesInOctalFields(string folderName, string testCaseName) [InlineData("invalid-go17")] // Many octal fields are all zero chars [InlineData("issue11169")] // Checksum with null in the middle [InlineData("issue10968")] // Garbage chars - [InlineData("writer-big")] // The size field contains an euro char public void Throw_ArchivesWithRandomChars(string testCaseName) { using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", testCaseName); @@ -279,6 +278,16 @@ public void Throw_ArchivesWithRandomChars(string testCaseName) Assert.Throws(() => reader.GetNextEntry()); } + [Fact] + public void Throw_ArchiveIsShort() + { + // writer-big has a header for a 16G file but not its contents. + using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", "writer-big"); + using TarReader reader = new TarReader(archiveStream); + // MemoryStream throws when we try to change its Position past its Length. + Assert.Throws(() => reader.GetNextEntry()); + } + [Fact] public void GarbageEntryChecksumZeroReturnNull() { diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs index ae847e45ac2d3..325c6c916f605 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs @@ -56,16 +56,33 @@ protected void SetFifo(GnuTarEntry fifo) protected void SetGnuProperties(GnuTarEntry entry) { + // The octal format limits the representable range. + bool formatIsOctalOnly = entry.Format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + DateTimeOffset approxNow = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(6)); // ATime: Verify the default value was approximately "now" Assert.True(entry.AccessTime > approxNow); - Assert.Throws(() => entry.AccessTime = DateTimeOffset.MinValue); + if (formatIsOctalOnly) + { + Assert.Throws(() => entry.AccessTime = DateTimeOffset.MinValue); + } + else + { + entry.AccessTime = DateTimeOffset.MinValue; + } entry.AccessTime = TestAccessTime; // CTime: Verify the default value was approximately "now" Assert.True(entry.ChangeTime > approxNow); - Assert.Throws(() => entry.ChangeTime = DateTimeOffset.MinValue); + if (formatIsOctalOnly) + { + Assert.Throws(() => entry.ChangeTime = DateTimeOffset.MinValue); + } + else + { + entry.ChangeTime = DateTimeOffset.MinValue; + } entry.ChangeTime = TestChangeTime; } diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs index 498612de473d6..3afc9a1746da1 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs @@ -19,6 +19,9 @@ protected void SetPosixProperties(PosixTarEntry entry) private void SetBlockDeviceProperties(PosixTarEntry device) { + // The octal format limits the representable range. + bool formatIsOctalOnly = device.Format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + Assert.NotNull(device); Assert.Equal(TarEntryType.BlockDevice, device.EntryType); SetCommonProperties(device); @@ -27,18 +30,35 @@ private void SetBlockDeviceProperties(PosixTarEntry device) // DeviceMajor Assert.Equal(DefaultDeviceMajor, device.DeviceMajor); Assert.Throws(() => device.DeviceMajor = -1); - Assert.Throws(() => device.DeviceMajor = 2097152); + if (formatIsOctalOnly) + { + Assert.Throws(() => device.DeviceMajor = 2097152); + } + else + { + device.DeviceMajor = 2097152; + } device.DeviceMajor = TestBlockDeviceMajor; // DeviceMinor Assert.Equal(DefaultDeviceMinor, device.DeviceMinor); Assert.Throws(() => device.DeviceMinor = -1); - Assert.Throws(() => device.DeviceMinor = 2097152); + if (formatIsOctalOnly) + { + Assert.Throws(() => device.DeviceMinor = 2097152); + } + else + { + device.DeviceMinor = 2097152; + } device.DeviceMinor = TestBlockDeviceMinor; } private void SetCharacterDeviceProperties(PosixTarEntry device) { + // The octal format limits the representable range. + bool formatIsOctalOnly = device.Format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + Assert.NotNull(device); Assert.Equal(TarEntryType.CharacterDevice, device.EntryType); SetCommonProperties(device); @@ -47,13 +67,27 @@ private void SetCharacterDeviceProperties(PosixTarEntry device) // DeviceMajor Assert.Equal(DefaultDeviceMajor, device.DeviceMajor); Assert.Throws(() => device.DeviceMajor = -1); - Assert.Throws(() => device.DeviceMajor = 2097152); + if (formatIsOctalOnly) + { + Assert.Throws(() => device.DeviceMajor = 2097152); + } + else + { + device.DeviceMajor = 2097152; + } device.DeviceMajor = TestCharacterDeviceMajor; // DeviceMinor Assert.Equal(DefaultDeviceMinor, device.DeviceMinor); Assert.Throws(() => device.DeviceMinor = -1); - Assert.Throws(() => device.DeviceMinor = 2097152); + if (formatIsOctalOnly) + { + Assert.Throws(() => device.DeviceMinor = 2097152); + } + else + { + device.DeviceMinor = 2097152; + } device.DeviceMinor = TestCharacterDeviceMinor; } diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs index 3b5d4ca93e090..29a15217879ee 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs @@ -311,6 +311,9 @@ protected void SetCommonSymbolicLink(TarEntry symbolicLink) protected void SetCommonProperties(TarEntry entry, bool isDirectory = false) { + // The octal format limits the range. + bool formatIsOctalOnly = entry.Format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + // Length (Data is checked outside this method) Assert.Equal(0, entry.Length); @@ -329,7 +332,14 @@ protected void SetCommonProperties(TarEntry entry, bool isDirectory = false) DateTimeOffset approxNow = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(6)); Assert.True(entry.ModificationTime > approxNow); - Assert.Throws(() => entry.ModificationTime = DateTime.MinValue); // Minimum allowed is UnixEpoch, not MinValue + if (formatIsOctalOnly) + { + Assert.Throws(() => entry.ModificationTime = DateTime.MinValue); // Minimum allowed is UnixEpoch, not MinValue + } + else + { + entry.ModificationTime = DateTimeOffset.MinValue; + } entry.ModificationTime = TestModificationTime; // Name diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs index 334a27e51fa92..d8d4018c40b01 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs @@ -48,5 +48,80 @@ protected void VerifyGlobalExtendedAttributesEntry(TarEntry entry, Dictionary WriteIntField_TheoryData() + { + foreach (TarEntryFormat format in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Pax, TarEntryFormat.Gnu }) + { + // Min value. + yield return new object[] { format, 0 }; + + yield return new object[] { format, 1 }; + yield return new object[] { format, 42 }; + + // Max value octal. + yield return new object[] { format, 0x1FFFFF }; + + // These values do not fit the octal representation. + bool formatIsOctalOnly = format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + if (!formatIsOctalOnly) + { + // Max value property. + yield return new object[] { format, int.MaxValue }; + } + + } + } + + public static IEnumerable WriteTimeStampsWithFormats_TheoryData() + { + foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax }) + { + foreach (DateTimeOffset timestamp in GetWriteTimeStamps(entryFormat)) + { + yield return new object[] { entryFormat, timestamp }; + } + } + } + + public static IEnumerable WriteTimeStamp_Pax_TheoryData() + { + foreach (DateTimeOffset timestamp in GetWriteTimeStamps(TarEntryFormat.Pax)) + { + yield return new object[] { timestamp }; + } + } + + private static IEnumerable GetWriteTimeStamps(TarEntryFormat format) + { + // One second past Y2K38 + yield return new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); + + // Min value octal + yield return DateTimeOffset.UnixEpoch; + + // Max value 12-byte octal field. + yield return DateTimeOffset.UnixEpoch + new TimeSpan(0x1FFFFFFFF * TimeSpan.TicksPerSecond); + + // These values do not fit the octal representation. + bool formatIsOctalOnly = format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + if (!formatIsOctalOnly) + { + // Min value property. + yield return DateTimeOffset.MinValue; // This is not representable with the octal format. + + // One second past what a 12-byte field can store with octal representation + yield return DateTimeOffset.UnixEpoch + new TimeSpan((0x1FFFFFFFF + 1) * TimeSpan.TicksPerSecond); + + // Max value property. Everything below seconds is set to zero for test equality comparison. + yield return new DateTimeOffset(new DateTime(DateTime.MaxValue.Year, + DateTime.MaxValue.Month, + DateTime.MaxValue.Day, + DateTime.MaxValue.Hour, + DateTime.MaxValue.Minute, + DateTime.MaxValue.Second, + DateTime.MaxValue.Kind), TimeSpan.Zero); + } + } } } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs index 1e81fb7b1e8a0..006aace683716 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs @@ -375,92 +375,30 @@ public void Add_Empty_GlobalExtendedAttributes() } } - [Fact] - // Y2K38 will happen one second after "2038/19/01 03:14:07 +00:00". This timestamp represents the seconds since the Unix epoch with a - // value of int.MaxValue: 2,147,483,647. - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // All our entry types should survive the Epochalypse because we internally use long to represent the seconds since Unix epoch, not int. - // So if the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, which - // is way past int MaxValue, but still within the long limits. That number represents the date "2242/16/03 12:56:32 +00:00". - public void WriteTimestampsBeyondEpochalypseInPax() - { - DateTimeOffset epochalypse = new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); - string strEpochalypse = GetTimestampStringFromDateTimeOffset(epochalypse); - - Dictionary ea = new Dictionary() - { - { PaxEaATime, strEpochalypse }, - { PaxEaCTime, strEpochalypse } - }; - - PaxTarEntry entry = new PaxTarEntry(TarEntryType.Directory, "dir", ea); - - entry.ModificationTime = epochalypse; - Assert.Equal(epochalypse, entry.ModificationTime); - - Assert.Contains(PaxEaATime, entry.ExtendedAttributes); - DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaATime); - Assert.Equal(epochalypse, atime); - - Assert.Contains(PaxEaCTime, entry.ExtendedAttributes); - DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(epochalypse, ctime); - - using MemoryStream archiveStream = new MemoryStream(); - using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) - { - writer.WriteEntry(entry); - } - - archiveStream.Position = 0; - using (TarReader reader = new TarReader(archiveStream)) - { - PaxTarEntry readEntry = reader.GetNextEntry() as PaxTarEntry; - Assert.NotNull(readEntry); - - Assert.Equal(epochalypse, readEntry.ModificationTime); - - Assert.Contains(PaxEaATime, readEntry.ExtendedAttributes); - DateTimeOffset actualATime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaATime); - Assert.Equal(epochalypse, actualATime); - - Assert.Contains(PaxEaCTime, readEntry.ExtendedAttributes); - DateTimeOffset actualCTime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(epochalypse, actualCTime); - } - } - - [Fact] - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // We internally use long to represent the seconds since Unix epoch, not int. - // If the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, - // which represents the date "2242/03/16 12:56:32 +00:00". - // Pax should survive after this date because it stores the timestamps in the extended attributes dictionary - // without size restrictions. - public void WriteTimestampsBeyondOctalLimitInPax() + [Theory] + [MemberData(nameof(WriteTimeStamp_Pax_TheoryData))] + public void WriteTimestampsInPax(DateTimeOffset timestamp) { - DateTimeOffset overLimitTimestamp = new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); // One second past the octal limit - - string strOverLimitTimestamp = GetTimestampStringFromDateTimeOffset(overLimitTimestamp); + string strTimestamp = GetTimestampStringFromDateTimeOffset(timestamp); Dictionary ea = new Dictionary() { - { PaxEaATime, strOverLimitTimestamp }, - { PaxEaCTime, strOverLimitTimestamp } + { PaxEaATime, strTimestamp }, + { PaxEaCTime, strTimestamp } }; PaxTarEntry entry = new PaxTarEntry(TarEntryType.Directory, "dir", ea); - entry.ModificationTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, entry.ModificationTime); + entry.ModificationTime = timestamp; + Assert.Equal(timestamp, entry.ModificationTime); Assert.Contains(PaxEaATime, entry.ExtendedAttributes); DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaATime); - Assert.Equal(overLimitTimestamp, atime); + Assert.Equal(timestamp, atime); Assert.Contains(PaxEaCTime, entry.ExtendedAttributes); DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(overLimitTimestamp, ctime); + Assert.Equal(timestamp, ctime); using MemoryStream archiveStream = new MemoryStream(); using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) @@ -474,15 +412,15 @@ public void WriteTimestampsBeyondOctalLimitInPax() PaxTarEntry readEntry = reader.GetNextEntry() as PaxTarEntry; Assert.NotNull(readEntry); - Assert.Equal(overLimitTimestamp, readEntry.ModificationTime); + Assert.Equal(timestamp, readEntry.ModificationTime); Assert.Contains(PaxEaATime, readEntry.ExtendedAttributes); DateTimeOffset actualATime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaATime); - Assert.Equal(overLimitTimestamp, actualATime); + Assert.Equal(timestamp, actualATime); Assert.Contains(PaxEaCTime, readEntry.ExtendedAttributes); DateTimeOffset actualCTime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(overLimitTimestamp, actualCTime); + Assert.Equal(timestamp, actualCTime); } } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs index 487c6ab04dbcd..e20ad1b9617bf 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs @@ -214,31 +214,22 @@ public void ReadAndWriteMultipleGlobalExtendedAttributesEntries(TarEntryFormat f } } - // Y2K38 will happen one second after "2038/19/01 03:14:07 +00:00". This timestamp represents the seconds since the Unix epoch with a - // value of int.MaxValue: 2,147,483,647. - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // All our entry types should survive the Epochalypse because we internally use long to represent the seconds since Unix epoch, not int. - // So if the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, which - // is way past int MaxValue, but still within the long limits. That number represents the date "2242/16/03 12:56:32 +00:00". [Theory] - [InlineData(TarEntryFormat.V7)] - [InlineData(TarEntryFormat.Ustar)] - [InlineData(TarEntryFormat.Gnu)] - public void WriteTimestampsBeyondEpochalypse(TarEntryFormat format) + [MemberData(nameof(WriteTimeStampsWithFormats_TheoryData))] + public void WriteTimeStamps(TarEntryFormat format, DateTimeOffset timestamp) { - DateTimeOffset epochalypse = new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); // One second past Y2K38 TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); - entry.ModificationTime = epochalypse; - Assert.Equal(epochalypse, entry.ModificationTime); + entry.ModificationTime = timestamp; + Assert.Equal(timestamp, entry.ModificationTime); if (entry is GnuTarEntry gnuEntry) { - gnuEntry.AccessTime = epochalypse; - Assert.Equal(epochalypse, gnuEntry.AccessTime); + gnuEntry.AccessTime = timestamp; + Assert.Equal(timestamp, gnuEntry.AccessTime); - gnuEntry.ChangeTime = epochalypse; - Assert.Equal(epochalypse, gnuEntry.ChangeTime); + gnuEntry.ChangeTime = timestamp; + Assert.Equal(timestamp, gnuEntry.ChangeTime); } using MemoryStream archiveStream = new MemoryStream(); @@ -253,43 +244,49 @@ public void WriteTimestampsBeyondEpochalypse(TarEntryFormat format) TarEntry readEntry = reader.GetNextEntry(); Assert.NotNull(readEntry); - Assert.Equal(epochalypse, readEntry.ModificationTime); + Assert.Equal(timestamp, readEntry.ModificationTime); if (readEntry is GnuTarEntry gnuReadEntry) { - Assert.Equal(epochalypse, gnuReadEntry.AccessTime); - Assert.Equal(epochalypse, gnuReadEntry.ChangeTime); + Assert.Equal(timestamp, gnuReadEntry.AccessTime); + Assert.Equal(timestamp, gnuReadEntry.ChangeTime); } } } - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // We internally use long to represent the seconds since Unix epoch, not int. - // If the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, - // which represents the date "2242/03/16 12:56:32 +00:00". - // V7, Ustar and GNU would not survive after this date because they only have the fixed size fields to store timestamps. [Theory] - [InlineData(TarEntryFormat.V7)] - [InlineData(TarEntryFormat.Ustar)] - [InlineData(TarEntryFormat.Gnu)] - public void WriteTimestampsBeyondOctalLimit(TarEntryFormat format) + [MemberData(nameof(WriteIntField_TheoryData))] + public void WriteUid(TarEntryFormat format, int value) { - DateTimeOffset overLimitTimestamp = new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); // One second past the octal limit - TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); - // Before writing the entry, the timestamps should have no issue - entry.ModificationTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, entry.ModificationTime); + entry.Uid = value; + Assert.Equal(value, entry.Uid); - if (entry is GnuTarEntry gnuEntry) + using MemoryStream archiveStream = new MemoryStream(); + using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) { - gnuEntry.AccessTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, gnuEntry.AccessTime); + writer.WriteEntry(entry); + } - gnuEntry.ChangeTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, gnuEntry.ChangeTime); + archiveStream.Position = 0; + using (TarReader reader = new TarReader(archiveStream)) + { + TarEntry readEntry = reader.GetNextEntry(); + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.Uid); } + } + + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public void WriteGid(TarEntryFormat format, int value) + { + TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); + + entry.Gid = value; + Assert.Equal(value, entry.Gid); using MemoryStream archiveStream = new MemoryStream(); using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) @@ -303,14 +300,69 @@ public void WriteTimestampsBeyondOctalLimit(TarEntryFormat format) TarEntry readEntry = reader.GetNextEntry(); Assert.NotNull(readEntry); - // The timestamps get stored as '{1970-01-01 12:00:00 AM +00:00}' due to the +1 overflow - Assert.NotEqual(overLimitTimestamp, readEntry.ModificationTime); + Assert.Equal(value, readEntry.Gid); + } + } - if (readEntry is GnuTarEntry gnuReadEntry) - { - Assert.NotEqual(overLimitTimestamp, gnuReadEntry.AccessTime); - Assert.NotEqual(overLimitTimestamp, gnuReadEntry.ChangeTime); - } + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public void WriteDeviceMajor(TarEntryFormat format, int value) + { + if (format == TarEntryFormat.V7) + { + return; // No DeviceMajor + } + + PosixTarEntry? entry = InvokeTarEntryCreationConstructor(format, TarEntryType.BlockDevice, "dir") as PosixTarEntry; + Assert.NotNull(entry); + + entry.DeviceMajor = value; + Assert.Equal(value, entry.DeviceMajor); + + using MemoryStream archiveStream = new MemoryStream(); + using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) + { + writer.WriteEntry(entry); + } + + archiveStream.Position = 0; + using (TarReader reader = new TarReader(archiveStream)) + { + PosixTarEntry? readEntry = reader.GetNextEntry() as PosixTarEntry; + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.DeviceMajor); + } + } + + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public void WriteDeviceMinor(TarEntryFormat format, int value) + { + if (format == TarEntryFormat.V7) + { + return; // No DeviceMinor + } + + PosixTarEntry? entry = InvokeTarEntryCreationConstructor(format, TarEntryType.BlockDevice, "dir") as PosixTarEntry; + Assert.NotNull(entry); + + entry.DeviceMinor = value; + Assert.Equal(value, entry.DeviceMinor); + + using MemoryStream archiveStream = new MemoryStream(); + using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) + { + writer.WriteEntry(entry); + } + + archiveStream.Position = 0; + using (TarReader reader = new TarReader(archiveStream)) + { + PosixTarEntry? readEntry = reader.GetNextEntry() as PosixTarEntry; + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.DeviceMinor); } } @@ -505,29 +557,6 @@ public void WriteEntry_UsingTarEntry_FromTarReader_IntoTarWriter(TarEntryFormat AssertExtensions.SequenceEqual(msSource.ToArray(), msDestination.ToArray()); } - [Theory] - [InlineData(TarEntryFormat.V7, false)] - [InlineData(TarEntryFormat.Ustar, false)] - [InlineData(TarEntryFormat.Gnu, false)] - [InlineData(TarEntryFormat.V7, true)] - [InlineData(TarEntryFormat.Ustar, true)] - [InlineData(TarEntryFormat.Gnu, true)] - public void WriteEntry_FileSizeOverLegacyLimit_Throws(TarEntryFormat entryFormat, bool unseekableStream) - { - const long FileSizeOverLimit = LegacyMaxFileSize + 1; - - using MemoryStream ms = new(); - using Stream s = unseekableStream ? new WrappedStream(ms, ms.CanRead, ms.CanWrite, canSeek: false) : ms; - - using TarWriter writer = new(s); - TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); - writeEntry.DataStream = new SimulatedDataStream(FileSizeOverLimit); - - Assert.Equal(FileSizeOverLimit, writeEntry.Length); - - Assert.Throws(() => writer.WriteEntry(writeEntry)); - } - [Theory] [InlineData(TarEntryFormat.V7)] [InlineData(TarEntryFormat.Ustar)] diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs index fc6445a3b3e97..b20d1c78e1aa0 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs @@ -397,92 +397,30 @@ public async Task Add_Empty_GlobalExtendedAttributes_Async() } } - [Fact] - // Y2K38 will happen one second after "2038/19/01 03:14:07 +00:00". This timestamp represents the seconds since the Unix epoch with a - // value of int.MaxValue: 2,147,483,647. - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // All our entry types should survive the Epochalypse because we internally use long to represent the seconds since Unix epoch, not int. - // So if the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, which - // is way past int MaxValue, but still within the long limits. That number represents the date "2242/16/03 12:56:32 +00:00". - public async Task WriteTimestampsBeyondEpochalypseInPax_Async() - { - DateTimeOffset epochalypse = new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); - string strEpochalypse = GetTimestampStringFromDateTimeOffset(epochalypse); - - Dictionary ea = new Dictionary() - { - { PaxEaATime, strEpochalypse }, - { PaxEaCTime, strEpochalypse } - }; - - PaxTarEntry entry = new PaxTarEntry(TarEntryType.Directory, "dir", ea); - - entry.ModificationTime = epochalypse; - Assert.Equal(epochalypse, entry.ModificationTime); - - Assert.Contains(PaxEaATime, entry.ExtendedAttributes); - DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaATime); - Assert.Equal(epochalypse, atime); - - Assert.Contains(PaxEaCTime, entry.ExtendedAttributes); - DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(epochalypse, ctime); - - using MemoryStream archiveStream = new MemoryStream(); - await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) - { - await writer.WriteEntryAsync(entry); - } - - archiveStream.Position = 0; - await using (TarReader reader = new TarReader(archiveStream)) - { - PaxTarEntry readEntry = await reader.GetNextEntryAsync() as PaxTarEntry; - Assert.NotNull(readEntry); - - Assert.Equal(epochalypse, readEntry.ModificationTime); - - Assert.Contains(PaxEaATime, readEntry.ExtendedAttributes); - DateTimeOffset actualATime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaATime); - Assert.Equal(epochalypse, actualATime); - - Assert.Contains(PaxEaCTime, readEntry.ExtendedAttributes); - DateTimeOffset actualCTime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(epochalypse, actualCTime); - } - } - - [Fact] - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // We internally use long to represent the seconds since Unix epoch, not int. - // If the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, - // which represents the date "2242/03/16 12:56:32 +00:00". - // Pax should survive after this date because it stores the timestamps in the extended attributes dictionary - // without size restrictions. - public async Task WriteTimestampsBeyondOctalLimitInPax_Async() + [Theory] + [MemberData(nameof(WriteTimeStamp_Pax_TheoryData))] + public async Task WriteTimestampsInPax_Async(DateTimeOffset timestamp) { - DateTimeOffset overLimitTimestamp = new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); // One second past the octal limit - - string strOverLimitTimestamp = GetTimestampStringFromDateTimeOffset(overLimitTimestamp); + string strTimestamp = GetTimestampStringFromDateTimeOffset(timestamp); Dictionary ea = new Dictionary() { - { PaxEaATime, strOverLimitTimestamp }, - { PaxEaCTime, strOverLimitTimestamp } + { PaxEaATime, strTimestamp }, + { PaxEaCTime, strTimestamp } }; PaxTarEntry entry = new PaxTarEntry(TarEntryType.Directory, "dir", ea); - entry.ModificationTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, entry.ModificationTime); + entry.ModificationTime = timestamp; + Assert.Equal(timestamp, entry.ModificationTime); Assert.Contains(PaxEaATime, entry.ExtendedAttributes); DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaATime); - Assert.Equal(overLimitTimestamp, atime); + Assert.Equal(timestamp, atime); Assert.Contains(PaxEaCTime, entry.ExtendedAttributes); DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(overLimitTimestamp, ctime); + Assert.Equal(timestamp, ctime); using MemoryStream archiveStream = new MemoryStream(); await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) @@ -496,15 +434,15 @@ public async Task WriteTimestampsBeyondOctalLimitInPax_Async() PaxTarEntry readEntry = await reader.GetNextEntryAsync() as PaxTarEntry; Assert.NotNull(readEntry); - Assert.Equal(overLimitTimestamp, readEntry.ModificationTime); + Assert.Equal(timestamp, readEntry.ModificationTime); Assert.Contains(PaxEaATime, readEntry.ExtendedAttributes); DateTimeOffset actualATime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaATime); - Assert.Equal(overLimitTimestamp, actualATime); + Assert.Equal(timestamp, actualATime); Assert.Contains(PaxEaCTime, readEntry.ExtendedAttributes); DateTimeOffset actualCTime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(overLimitTimestamp, actualCTime); + Assert.Equal(timestamp, actualCTime); } } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs index 45b26ad66e600..f22b22dc63d1c 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs @@ -237,31 +237,22 @@ public async Task ReadAndWriteMultipleGlobalExtendedAttributesEntries_Async(TarE } } - // Y2K38 will happen one second after "2038/19/01 03:14:07 +00:00". This timestamp represents the seconds since the Unix epoch with a - // value of int.MaxValue: 2,147,483,647. - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // All our entry types should survive the Epochalypse because we internally use long to represent the seconds since Unix epoch, not int. - // So if the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, which - // is way past int MaxValue, but still within the long limits. That number represents the date "2242/16/03 12:56:32 +00:00". [Theory] - [InlineData(TarEntryFormat.V7)] - [InlineData(TarEntryFormat.Ustar)] - [InlineData(TarEntryFormat.Gnu)] - public async Task WriteTimestampsBeyondEpochalypse_Async(TarEntryFormat format) + [MemberData(nameof(WriteTimeStampsWithFormats_TheoryData))] + public async Task WriteTimeStamps_Async(TarEntryFormat format, DateTimeOffset timestamp) { - DateTimeOffset epochalypse = new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); // One second past Y2K38 TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); - entry.ModificationTime = epochalypse; - Assert.Equal(epochalypse, entry.ModificationTime); + entry.ModificationTime = timestamp; + Assert.Equal(timestamp, entry.ModificationTime); if (entry is GnuTarEntry gnuEntry) { - gnuEntry.AccessTime = epochalypse; - Assert.Equal(epochalypse, gnuEntry.AccessTime); + gnuEntry.AccessTime = timestamp; + Assert.Equal(timestamp, gnuEntry.AccessTime); - gnuEntry.ChangeTime = epochalypse; - Assert.Equal(epochalypse, gnuEntry.ChangeTime); + gnuEntry.ChangeTime = timestamp; + Assert.Equal(timestamp, gnuEntry.ChangeTime); } using MemoryStream archiveStream = new MemoryStream(); @@ -276,43 +267,49 @@ public async Task WriteTimestampsBeyondEpochalypse_Async(TarEntryFormat format) TarEntry readEntry = await reader.GetNextEntryAsync(); Assert.NotNull(readEntry); - Assert.Equal(epochalypse, readEntry.ModificationTime); + Assert.Equal(timestamp, readEntry.ModificationTime); if (readEntry is GnuTarEntry gnuReadEntry) { - Assert.Equal(epochalypse, gnuReadEntry.AccessTime); - Assert.Equal(epochalypse, gnuReadEntry.ChangeTime); + Assert.Equal(timestamp, gnuReadEntry.AccessTime); + Assert.Equal(timestamp, gnuReadEntry.ChangeTime); } } } - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // We internally use long to represent the seconds since Unix epoch, not int. - // If the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, - // which represents the date "2242/03/16 12:56:32 +00:00". - // V7, Ustar and GNU would not survive after this date because they only have the fixed size fields to store timestamps. [Theory] - [InlineData(TarEntryFormat.V7)] - [InlineData(TarEntryFormat.Ustar)] - [InlineData(TarEntryFormat.Gnu)] - public async Task WriteTimestampsBeyondOctalLimit_Async(TarEntryFormat format) + [MemberData(nameof(WriteIntField_TheoryData))] + public async Task WriteUid_Async(TarEntryFormat format, int value) { - DateTimeOffset overLimitTimestamp = new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); // One second past the octal limit - TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); - // Before writing the entry, the timestamps should have no issue - entry.ModificationTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, entry.ModificationTime); + entry.Uid = value; + Assert.Equal(value, entry.Uid); - if (entry is GnuTarEntry gnuEntry) + using MemoryStream archiveStream = new MemoryStream(); + await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) { - gnuEntry.AccessTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, gnuEntry.AccessTime); + await writer.WriteEntryAsync(entry); + } - gnuEntry.ChangeTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, gnuEntry.ChangeTime); + archiveStream.Position = 0; + await using (TarReader reader = new TarReader(archiveStream)) + { + TarEntry readEntry = await reader.GetNextEntryAsync(); + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.Uid); } + } + + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public async Task WriteGid_Async(TarEntryFormat format, int value) + { + TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); + + entry.Gid = value; + Assert.Equal(value, entry.Gid); using MemoryStream archiveStream = new MemoryStream(); await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) @@ -326,14 +323,69 @@ public async Task WriteTimestampsBeyondOctalLimit_Async(TarEntryFormat format) TarEntry readEntry = await reader.GetNextEntryAsync(); Assert.NotNull(readEntry); - // The timestamps get stored as '{1970-01-01 12:00:00 AM +00:00}' due to the +1 overflow - Assert.NotEqual(overLimitTimestamp, readEntry.ModificationTime); + Assert.Equal(value, readEntry.Gid); + } + } - if (readEntry is GnuTarEntry gnuReadEntry) - { - Assert.NotEqual(overLimitTimestamp, gnuReadEntry.AccessTime); - Assert.NotEqual(overLimitTimestamp, gnuReadEntry.ChangeTime); - } + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public async Task WriteDeviceMajor_Async(TarEntryFormat format, int value) + { + if (format == TarEntryFormat.V7) + { + return; // No DeviceMajor + } + + PosixTarEntry? entry = InvokeTarEntryCreationConstructor(format, TarEntryType.BlockDevice, "dir") as PosixTarEntry; + Assert.NotNull(entry); + + entry.DeviceMajor = value; + Assert.Equal(value, entry.DeviceMajor); + + using MemoryStream archiveStream = new MemoryStream(); + await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) + { + await writer.WriteEntryAsync(entry); + } + + archiveStream.Position = 0; + await using (TarReader reader = new TarReader(archiveStream)) + { + PosixTarEntry? readEntry = await reader.GetNextEntryAsync() as PosixTarEntry; + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.DeviceMajor); + } + } + + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public async Task WriteDeviceMinor_Async(TarEntryFormat format, int value) + { + if (format == TarEntryFormat.V7) + { + return; // No DeviceMinor + } + + PosixTarEntry? entry = InvokeTarEntryCreationConstructor(format, TarEntryType.BlockDevice, "dir") as PosixTarEntry; + Assert.NotNull(entry); + + entry.DeviceMinor = value; + Assert.Equal(value, entry.DeviceMinor); + + using MemoryStream archiveStream = new MemoryStream(); + await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) + { + await writer.WriteEntryAsync(entry); + } + + archiveStream.Position = 0; + await using (TarReader reader = new TarReader(archiveStream)) + { + PosixTarEntry? readEntry = await reader.GetNextEntryAsync() as PosixTarEntry; + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.DeviceMinor); } } @@ -423,30 +475,6 @@ public async Task WriteEntry_UsingTarEntry_FromTarReader_IntoTarWriter_Async(Tar AssertExtensions.SequenceEqual(msSource.ToArray(), msDestination.ToArray()); } - [Theory] - [InlineData(TarEntryFormat.V7, false)] - [InlineData(TarEntryFormat.Ustar, false)] - [InlineData(TarEntryFormat.Gnu, false)] - [InlineData(TarEntryFormat.V7, true)] - [InlineData(TarEntryFormat.Ustar, true)] - [InlineData(TarEntryFormat.Gnu, true)] - public async Task WriteEntry_FileSizeOverLegacyLimit_Throws_Async(TarEntryFormat entryFormat, bool unseekableStream) - { - const long FileSizeOverLimit = LegacyMaxFileSize + 1; - - await using MemoryStream ms = new(); - await using Stream s = unseekableStream ? new WrappedStream(ms, ms.CanRead, ms.CanWrite, canSeek: false) : ms; - - string tarFilePath = GetTestFilePath(); - await using TarWriter writer = new(File.Create(tarFilePath)); - TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); - writeEntry.DataStream = new SimulatedDataStream(FileSizeOverLimit); - - Assert.Equal(FileSizeOverLimit, writeEntry.Length); - - await Assert.ThrowsAsync(() => writer.WriteEntryAsync(writeEntry)); - } - [Theory] [InlineData(TarEntryFormat.V7)] [InlineData(TarEntryFormat.Ustar)]