Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixed jackson transient serialization bug and handled HTTP status 204 in HTTPResponseWrapper #310

Merged
merged 9 commits into from
Jul 17, 2023
3 changes: 2 additions & 1 deletion data/src/main/java/com/microsoft/azure/kusto/data/Utils.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
Expand Down Expand Up @@ -59,7 +60,7 @@ public class Utils {
// added auto bigdecimal deserialization for float and double value, since the bigdecimal values seem to loose precision while auto deserialization to
// double value
public static ObjectMapper getObjectMapper() {
return JsonMapper.builder().addModule(new JavaTimeModule()).build().configure(
return JsonMapper.builder().configure(MapperFeature.PROPAGATE_TRANSIENT_MARKER, true).addModule(new JavaTimeModule()).build().configure(
DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true).configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true).setNodeFactory(
JsonNodeFactory.withExactBigDecimals(true));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.microsoft.aad.msal4j.IHttpResponse;

import org.apache.http.Header;
import org.apache.http.HttpStatus;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
Expand Down Expand Up @@ -82,7 +83,11 @@ public Mono<byte[]> getBodyAsByteArray() {

if (body == null) {
try {
body = EntityUtils.toByteArray(response.getEntity());
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_NO_CONTENT) {
body = new byte[0];
tanmaya-panda1 marked this conversation as resolved.
Show resolved Hide resolved
} else {
body = EntityUtils.toByteArray(response.getEntity());
}
} catch (IOException ignored) {
body = new byte[0];
}
Expand Down
61 changes: 47 additions & 14 deletions ingest/src/test/java/com/microsoft/azure/kusto/ingest/E2ETest.java
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,11 @@ private static void createTestData() {
ingestionPropertiesWithColumnMapping.setIngestionMapping(columnMapping, IngestionMappingKind.JSON);
ingestionPropertiesWithColumnMapping.setDataFormat(DataFormat.JSON);

IngestionProperties ingestionPropertiesWithTableReportMethod = new IngestionProperties(databaseName, tableName);
ingestionPropertiesWithTableReportMethod.setFlushImmediately(true);
ingestionPropertiesWithTableReportMethod.setDataFormat(DataFormat.CSV);
ingestionPropertiesWithTableReportMethod.setReportMethod(IngestionProperties.IngestionReportMethod.TABLE);

dataForTests = Arrays.asList(new TestDataItem() {
{
file = new File(resourcesPath, "dataset.csv");
Expand Down Expand Up @@ -249,6 +254,15 @@ private static void createTestData() {
ingestionProperties = ingestionPropertiesWithColumnMapping;
testOnstreamingIngestion = false; // streaming ingestion doesn't support inline mapping
}
}, new TestDataItem() {
{
file = new File(resourcesPath, "dataset.csv");
rows = 10;
ingestionProperties = ingestionPropertiesWithTableReportMethod;
testOnstreamingIngestion = false;
testOnManaged = false;
testReportMethodTable = true;
}
});
}

Expand Down Expand Up @@ -321,31 +335,50 @@ private boolean isDatabasePrincipal(Client localQueryClient) {
@ValueSource(booleans = {true, false})
void testIngestFromFile(boolean isManaged) {
for (TestDataItem item : dataForTests) {
FileSourceInfo fileSourceInfo = new FileSourceInfo(item.file.getPath(), item.file.length());
try {
((isManaged && item.testOnManaged) ? managedStreamingIngestClient : ingestClient).ingestFromFile(fileSourceInfo, item.ingestionProperties);
} catch (Exception ex) {
Assertions.fail(ex);
if (!item.testReportMethodTable) {
FileSourceInfo fileSourceInfo = new FileSourceInfo(item.file.getPath(), item.file.length());
try {
((isManaged && item.testOnManaged) ? managedStreamingIngestClient : ingestClient).ingestFromFile(fileSourceInfo, item.ingestionProperties);
} catch (Exception ex) {
Assertions.fail(ex);
}
assertRowCount(item.rows, false);
}
}
}

@Test
void testIngestFromFileWithTable() {
for (TestDataItem item : dataForTests) {
if (item.testReportMethodTable) {
FileSourceInfo fileSourceInfo = new FileSourceInfo(item.file.getPath(), item.file.length());
try {
ingestClient.ingestFromFile(fileSourceInfo, item.ingestionProperties);
} catch (Exception ex) {
Assertions.fail(ex);
}
assertRowCount(item.rows, false);
}
assertRowCount(item.rows, false);
}
}

@ParameterizedTest
@ValueSource(booleans = {true, false})
void testIngestFromStream(boolean isManaged) throws FileNotFoundException {
for (TestDataItem item : dataForTests) {
InputStream stream = new FileInputStream(item.file);
StreamSourceInfo streamSourceInfo = new StreamSourceInfo(stream);
if (item.file.getPath().endsWith(".gz")) {
InputStream stream = new FileInputStream(item.file);
StreamSourceInfo streamSourceInfo = new StreamSourceInfo(stream);

streamSourceInfo.setCompressionType(CompressionType.gz);
try {
((isManaged && item.testOnManaged) ? managedStreamingIngestClient : ingestClient).ingestFromStream(streamSourceInfo,
item.ingestionProperties);
} catch (Exception ex) {
Assertions.fail(ex);
}
assertRowCount(item.rows, true);
}
try {
((isManaged && item.testOnManaged) ? managedStreamingIngestClient : ingestClient).ingestFromStream(streamSourceInfo, item.ingestionProperties);
} catch (Exception ex) {
Assertions.fail(ex);
}
assertRowCount(item.rows, true);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ public class TestDataItem {
public int rows;
public boolean testOnstreamingIngestion = true;
public boolean testOnManaged = true;
public boolean testReportMethodTable = false;
}
Loading