Updated DeIdentification samples and tests.

kurtisvg · kurtisvg · commit 184c006adee3 · 2018-03-13T13:46:19.000-07:00
diff --git a/dlp/src/main/java/com/example/dlp/DeIdentification.java b/dlp/src/main/java/com/example/dlp/DeIdentification.java
@@ -77,9 +77,7 @@ public class DeIdentification {
    * @param projectId ID of Google Cloud project to run the API under.
    */
   private static void deIdentifyWithMask(
-      String string,
-      Character maskingCharacter,
-      int numberToMask) {
+      String string, Character maskingCharacter, int numberToMask, String projectId) {
     // [START dlp_deidentify_masking]
     /**
      * Deidentify a string by masking sensitive information with a character using the DLP API.
@@ -95,7 +93,6 @@ private static void deIdentifyWithMask(
       // string = "My SSN is 372819127";
       // numberToMask = 5;
       // maskingCharacter = 'x';
-
       ByteContentItem byteContentItem =
           ByteContentItem.newBuilder()
               .setType(ByteContentItem.BytesType.TEXT_UTF8)
@@ -112,7 +109,9 @@ private static void deIdentifyWithMask(
 
       // Create the deidentification transformation configuration
       PrimitiveTransformation primitiveTransformation =
-          PrimitiveTransformation.newBuilder().setCharacterMaskConfig(characterMaskConfig).build();
+          PrimitiveTransformation.newBuilder()
+              .setCharacterMaskConfig(characterMaskConfig)
+              .build();
 
       InfoTypeTransformation infoTypeTransformationObject =
           InfoTypeTransformation.newBuilder()
@@ -124,15 +123,15 @@ private static void deIdentifyWithMask(
               .addTransformations(infoTypeTransformationObject)
               .build();
 
-      // Create the deidentification request object
       DeidentifyConfig deidentifyConfig =
           DeidentifyConfig.newBuilder()
               .setInfoTypeTransformations(infoTypeTransformationArray)
               .build();
 
+      // Create the deidentification request object
       DeidentifyContentRequest request =
           DeidentifyContentRequest.newBuilder()
-              .setParent(projectId)
+              .setParent(String.format("projects/%s", projectId))
               .setDeidentifyConfig(deidentifyConfig)
               .setItem(contentItem)
               .build();
@@ -142,10 +141,11 @@ private static void deIdentifyWithMask(
 
       // Print the character-masked input value
       // e.g. "My SSN is 123456789" --> "My SSN is *********"
-      ContentItem item = response.getItem();
-      System.out.println(item.getValue());
+      String result = response.getItem().getByteItem().getData().toStringUtf8();
+      System.out.println(result);
     } catch (Exception e) {
       System.out.println("Error in deidentifyWithMask: " + e.getMessage());
+      System.out.println(e.getStackTrace());
     }
   }
   // [END dlp_deidentify_mask]
@@ -223,7 +223,7 @@ private static void deIdentifyWithFpe(
 
       DeidentifyContentRequest request =
           DeidentifyContentRequest.newBuilder()
-              .setParent(projectId)
+              .setParent(String.format("projects/%s", projectId))
               .setDeidentifyConfig(deidentifyConfig)
               .setItem(contentItem)
               .build();
@@ -233,8 +233,8 @@ private static void deIdentifyWithFpe(
 
       // Print the deidentified input value
       // e.g. "My SSN is 123456789" --> "My SSN is 7261298621"
-      ContentItem item = response.getItem();
-      System.out.println(item.getValue());
+      String result = response.getItem().getByteItem().getData().toStringUtf8();
+      System.out.println(result);
     } catch (Exception e) {
       System.out.println("Error in deidentifyWithFpe: " + e.getMessage());
     }
@@ -292,7 +292,7 @@ private static void deidentifyWithDateShift(
         KmsWrappedCryptoKey kmsWrappedCryptoKey =
             KmsWrappedCryptoKey.newBuilder()
                 .setCryptoKeyName(keyName)
-                .setWrappedKey(ByteString.copyFromUtf8(wrappedKey))
+                .setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedKey)))
                 .build();
         dateShiftConfigBuilder.setCryptoKey(
             CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build());
@@ -352,7 +352,7 @@ private static void deidentifyWithDateShift(
 
       DeidentifyContentRequest request =
           DeidentifyContentRequest.newBuilder()
-              .setParent(projectId)
+              .setParent(String.format("projects/%s", projectId))
               .setDeidentifyConfig(deidentifyConfig)
               .setItem(tableItem)
               .build();
@@ -369,7 +369,7 @@ private static void deidentifyWithDateShift(
 
       File outputFile = outputCsvPath.toFile();
       if (!outputFile.exists()) {
-        outputFile.mkdirs();
+        outputFile.getParentFile().mkdirs();
         outputFile.createNewFile();
       }
       BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(outputFile));
@@ -379,27 +379,19 @@ private static void deidentifyWithDateShift(
 
       // write out each row
       for (Table.Row outputRow : outputRows) {
-        String row =
-            outputRow
-                .getValuesList()
-                .stream()
-                .map(
-                    value ->
-                        (value.getDateValue() != null)
-                            ? (String.valueOf(value.getDateValue().getMonth())
-                                + "/"
-                                + String.valueOf(value.getDateValue().getDay())
-                                + "/"
-                                + String.valueOf(value.getDateValue().getYear()))
-                            : value.getStringValue())
-                .collect(Collectors.joining(","));
+        String row = outputRow.getValuesList()
+            .stream()
+            .map(value -> value.getStringValue())
+            .collect(Collectors.joining(","));
         bufferedWriter.append(row + "\n");
       }
 
       bufferedWriter.flush();
       bufferedWriter.close();
 
-      System.out.println("Successfully saved date-shift output to:" + outputCsvPath.getFileName());
+      System.out.println("Successfully saved date-shift output to: " + outputCsvPath.getFileName());
+    } catch (Exception e) {
+      System.out.println("Error in deidentifyWithDateShift: " + e.getMessage());
     }
   }
 
@@ -446,15 +438,16 @@ public static void main(String[] args) throws Exception {
     optionsGroup.setRequired(true);
 
     Option deidentifyMaskingOption =
-        new Option("m", "mask", true, "Deidentify with character masking");
+        new Option("m", "mask", true, "Deidentify with character masking.");
     optionsGroup.addOption(deidentifyMaskingOption);
 
-    Option deidentifyFpeOption = new Option("f", "fpe", true, "Deidentify with FFX FPE");
+    Option deidentifyFpeOption =
+        new Option("f", "fpe", true, "Deidentify with FFX FPE.");
     optionsGroup.addOption(deidentifyFpeOption);
 
     Option deidentifyDateShiftOption =
-        new Option(
-            "d", "date", true, "Deidentify dates in a CSV file by pseudorandomly shifting them.");
+        new Option("d", "date", false, "Deidentify dates in a CSV file.");
+    optionsGroup.addOption(deidentifyDateShiftOption);
 
     Options commandLineOptions = new Options();
     commandLineOptions.addOptionGroup(optionsGroup);
@@ -533,8 +526,8 @@ public static void main(String[] args) throws Exception {
               cmd.getOptionValue(
                   alphabetOption.getOpt(), FfxCommonNativeAlphabet.ALPHA_NUMERIC.name()));
       deIdentifyWithFpe(val, alphabet, keyName, wrappedKey, projectId);
-    } // deidentify with date shift
-    else if (cmd.hasOption("d")) {
+    } else if (cmd.hasOption("d")) {
+      //deidentify with date shift
       String inputCsv = cmd.getOptionValue(inputCsvPathOption.getOpt());
       String outputCsv = cmd.getOptionValue(outputCsvPathOption.getOpt());
 
diff --git a/dlp/src/test/java/com/example/dlp/DeIdentificationIT.java b/dlp/src/test/java/com/example/dlp/DeIdentificationIT.java
@@ -16,7 +16,8 @@
 
 package com.example.dlp;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -25,6 +26,7 @@
 import java.io.PrintStream;
 import java.util.regex.Pattern;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -64,7 +66,7 @@ public void testDeidStringMasksCharacters() throws Exception {
           "-numberToMask", "5"
         });
     String output = bout.toString();
-    assertEquals(output, "My SSN is xxxxx9127\n");
+    assertThat(output, containsString("My SSN is xxxxx9127"));
   }
 
   @Test
@@ -77,8 +79,29 @@ public void testDeidStringPerformsFpe() throws Exception {
           "-keyName", keyName
         });
     String output = bout.toString();
-    assertFalse(output.contains(text));
-    assertTrue(Pattern.compile("My SSN is \\w+").matcher(output).find());
+    assertFalse(
+        "Response contains original SSN.",
+        output.contains("372819127"));
+    assertThat(output, containsString("My SSN is "));
+  }
+
+  @Test
+  public void testDeidentifyWithDateShift() throws Exception {
+    DeIdentification.main(
+        new String[] {
+            "-d",
+            "-inputCsvPath", "src/test/resources/dates.csv",
+            "-outputCsvPath", "src/test/resources/results.temp.csv",
+           "-dateFields", "birth_date,register_date",
+            "-lowerBoundDays", "5",
+            "-upperBoundDays", "5",
+            "-contextField", "name",
+            "-wrappedKey", wrappedKey,
+            "-keyName", keyName
+        });
+    String output = bout.toString();
+    assertThat(
+        output, containsString("Successfully saved date-shift output to: results.temp.csv"));
   }
 
   @After
diff --git a/dlp/src/test/resources/dates.csv b/dlp/src/test/resources/dates.csv
