From fd5a845390585e0341ded2bf97a44dcbb68ee4a5 Mon Sep 17 00:00:00 2001 From: Alexander Fink Date: Tue, 22 May 2018 10:59:55 +0200 Subject: [PATCH] Implemented fine-grained merging using annotations (#4) - fixes #1 by merging only Codelib functions annotated with a certain annotation type - removed confusing error messages - fixed a problem when unpacking apks containing filenames with invalid UTF-8 characters --- src/main/java/comm/android/dex/Dex.java | 22 + .../java/comm/android/dx/merge/DexMerger.java | 384 ++++++++++++----- .../java/comm/android/dx/merge/IndexMap.java | 51 ++- .../dx/merge/InstructionTransformer.java | 5 + .../comm/android/dx/merge/MethodFilter.java | 408 ++++++++++++++++++ .../cispa/apksigner/zipio/ZipInput.java | 2 +- .../cispa/apksigner/zipio/ZipOutput.java | 2 +- .../saarland/cispa/dexterous/Dexterous.java | 31 +- .../saarland/cispa/dexterous/MultiDex.java | 7 +- .../cispa/dexterous/cli/Dexterously.java | 19 +- .../saarland/cispa/dexterous/cli/Main.java | 22 +- 11 files changed, 801 insertions(+), 152 deletions(-) create mode 100644 src/main/java/comm/android/dx/merge/MethodFilter.java diff --git a/src/main/java/comm/android/dex/Dex.java b/src/main/java/comm/android/dex/Dex.java index ad25314..0bcc4aa 100644 --- a/src/main/java/comm/android/dex/Dex.java +++ b/src/main/java/comm/android/dex/Dex.java @@ -22,6 +22,8 @@ import comm.android.dex.util.ByteOutput; import comm.android.dex.util.FileUtils; import comm.android.dex.util.ByteOutput; +import comm.android.dx.merge.DexMerger; +import comm.android.dx.merge.MethodFilter; import java.io.ByteArrayOutputStream; import java.io.File; @@ -69,6 +71,17 @@ public final class Dex { private final FieldIdTable fieldIds = new FieldIdTable(); private final MethodIdTable methodIds = new MethodIdTable(); + private MethodFilter methodFilter; + + { + try { + methodFilter = new MethodFilter(this, null); + } catch (DexMerger.MergeException e) { + // should never happen + e.printStackTrace(); + } + } + /** * Creates a new dex that reads from {@code data}. It is an error to modify * {@code data} after using it to create a dex buffer. @@ -337,6 +350,14 @@ public void setName(final String dexName) { this.dexName = dexName; } + public MethodFilter getMethodFilter() { + return methodFilter; + } + + public void setWhitelistedAnnotation(String s) throws DexMerger.MergeException { + this.methodFilter = new MethodFilter(this, s); + } + public final class Section implements ByteInput, ByteOutput { private final String name; private final ByteBuffer data; @@ -592,6 +613,7 @@ private byte[] getBytesFrom(int start) { } public Annotation readAnnotation() { + //Log.w("SECTION", data.position()+"/"+ data.limit()); byte visibility = readByte(); int start = data.position(); new EncodedValueReader(this, EncodedValueReader.ENCODED_ANNOTATION).skipValue(); diff --git a/src/main/java/comm/android/dx/merge/DexMerger.java b/src/main/java/comm/android/dx/merge/DexMerger.java index 8ae3dd7..dd634e0 100644 --- a/src/main/java/comm/android/dx/merge/DexMerger.java +++ b/src/main/java/comm/android/dx/merge/DexMerger.java @@ -23,6 +23,7 @@ import comm.android.dex.Dex; import comm.android.dex.DexException; import comm.android.dex.DexIndexOverflowException; +import comm.android.dex.EncodedValueReader; import comm.android.dex.FieldId; import comm.android.dex.MethodId; import comm.android.dex.ProtoId; @@ -30,14 +31,9 @@ import comm.android.dex.TableOfContents; import comm.android.dex.TypeList; import comm.android.dx.command.dexer.DxContext; -import comm.android.dex.Annotation; -import comm.android.dex.Code; -import comm.android.dex.SizeOf; -import comm.android.dx.command.dexer.DxContext; import saarland.cispa.utils.LogUtils; import trikita.log.Log; -import java.io.File; import java.io.IOException; import java.util.*; @@ -91,6 +87,9 @@ public final class DexMerger { private final InstructionTransformer instructionTransformer; private final String codelibDexName; + public static final int OFFSET_BLACKLISTED = -2; + public static final int INDEX_BLACKLISTED = -1; + /** minimum number of wasted bytes before it's worthwhile to compact the result */ private int compactWasteThreshold = 1024 * 1024; // 1MiB @@ -234,16 +233,20 @@ public Dex merge() throws IOException { return result; } - public Dex mergeMethodsOnly() throws IOException { + public Dex mergeMethodsOnly() throws MergeException { if (dexes.length == 1) { return dexes[0]; } else if (dexes.length == 0) { return null; } - Dex result = mergeDexesMethods(); - - result = compactDex(result); + Dex result = null; + try { + result = mergeDexesMethods(); + result = compactDex(result); + } catch (Throwable e) { + throw new MergeException(e); + } return result; } @@ -293,6 +296,8 @@ protected IdMerger(Dex.Section out) { this.out = out; } + private final int FIRST_ITERATION = -3; + /** * Merges already-sorted sections, reading one value from each dex into memory * at a time. @@ -301,7 +306,10 @@ public final void mergeSorted() { TableOfContents.Section[] sections = new TableOfContents.Section[dexes.length]; Dex.Section[] dexSections = new Dex.Section[dexes.length]; int[] offsets = new int[dexes.length]; - int[] indexes = new int[dexes.length]; + Integer[] indexes = new Integer[dexes.length]; + for (int i = 0; i < dexes.length; i++) { + indexes[i] = 0; + } // values contains one value from each dex, sorted for fast retrieval of // the smallest value. The list associated with a value has the indexes @@ -312,8 +320,13 @@ public final void mergeSorted() { sections[i] = getSection(dexes[i].getTableOfContents()); dexSections[i] = sections[i].exists() ? dexes[i].open(sections[i].off) : null; // Fill in values with the first value of each dex. - offsets[i] = readIntoMap( - dexSections[i], sections[i], indexMaps[i], indexes[i], values, i); + for (offsets[i] = FIRST_ITERATION; offsets[i] == OFFSET_BLACKLISTED || offsets[i] == FIRST_ITERATION;) { + offsets[i] = readIntoMap(dexSections[i], sections[i], + indexMaps[i], indexes[i], values, i); + if (offsets[i] == OFFSET_BLACKLISTED) { + updateIndex(offsets[i], indexMaps[i], indexes[i]++, INDEX_BLACKLISTED); + } + } } if (values.isEmpty()) { getSection(contentsOut).off = 0; @@ -323,26 +336,36 @@ public final void mergeSorted() { getSection(contentsOut).off = out.getPosition(); int outCount = 0; + int origOutCount = 0; while (!values.isEmpty()) { Map.Entry> first = values.pollFirstEntry(); for (Integer dex : first.getValue()) { - updateIndex(offsets[dex], indexMaps[dex], indexes[dex]++, outCount); - // Fetch the next value of the dexes we just polled out - offsets[dex] = readIntoMap(dexSections[dex], sections[dex], - indexMaps[dex], indexes[dex], values, dex); + for (offsets[dex] = FIRST_ITERATION; offsets[dex] == OFFSET_BLACKLISTED || offsets[dex] == FIRST_ITERATION;) { + updateIndex(offsets[dex], indexMaps[dex], indexes[dex]++, offsets[dex] == FIRST_ITERATION ? outCount : INDEX_BLACKLISTED); + // Fetch the next value of the dexes we just polled out + offsets[dex] = readIntoMap(dexSections[dex], sections[dex], + indexMaps[dex], indexes[dex], values, dex); + if (dex == 1 && offsets[dex] == OFFSET_BLACKLISTED) { + origOutCount++; + } + } } write(first.getKey()); outCount++; } - + Log.d("Blacklisted: " + origOutCount); getSection(contentsOut).size = outCount; } + private int readIntoMap(Dex.Section in, TableOfContents.Section section, IndexMap indexMap, int index, TreeMap> values, int dex) { int offset = in != null ? in.getPosition() : -1; if (index < section.size) { - T v = read(in, indexMap, index); + T v = read(in, indexMap, index, dex); + if (v == null) { + return OFFSET_BLACKLISTED; + } List l = values.get(v); if (l == null) { l = new ArrayList(); @@ -374,15 +397,16 @@ public final void mergeUnsorted() { int outCount = 0; for (int i = 0; i < all.size(); ) { UnsortedValue e1 = all.get(i++); - updateIndex(e1.offset, e1.indexMap, e1.index, outCount - 1); + updateIndex(e1.offset, e1.indexMap, e1.index, e1.value == null ? INDEX_BLACKLISTED : outCount - 1); - while (i < all.size() && e1.compareTo(all.get(i)) == 0) { + while (i < all.size() && e1.value != null && e1.compareTo(all.get(i)) == 0) { UnsortedValue e2 = all.get(i++); - updateIndex(e2.offset, e2.indexMap, e2.index, outCount - 1); + updateIndex(e2.offset, e2.indexMap, e2.index, e2.value == null ? INDEX_BLACKLISTED : outCount - 1); + } + if (e1.value != null) { + write(e1.value); + outCount++; } - - write(e1.value); - outCount++; } getSection(contentsOut).size = outCount; @@ -405,7 +429,12 @@ private List readUnsortedValues(Dex source, IndexMap indexMap) { } abstract TableOfContents.Section getSection(TableOfContents tableOfContents); - abstract T read(Dex.Section in, IndexMap indexMap, int index); + T read(Dex.Section in, IndexMap indexMap, int index) { + throw new RuntimeException("not implemented"); + } + T read(Dex.Section in, IndexMap indexMap, int index, int dex) { + return read(in, indexMap, index); + } abstract void updateIndex(int offset, IndexMap indexMap, int oldIndex, int newIndex); abstract void write(T value); @@ -426,7 +455,15 @@ class UnsortedValue implements Comparable { @Override public int compareTo(UnsortedValue unsortedValue) { - return value.compareTo(unsortedValue.value); + if (value == null && unsortedValue.value == null) { + return 0; + } else if (value == null) { + return -1; + } else if (unsortedValue.value == null) { + return 1; + } else { + return value.compareTo(unsortedValue.value); + } } } } @@ -443,17 +480,31 @@ private int mergeApiLevels() { } private void mergeStringIds() { + Log.d("DexMerger","mergeStringIds..."); new IdMerger(idsDefsOut) { @Override TableOfContents.Section getSection(TableOfContents tableOfContents) { return tableOfContents.stringIds; } - @Override String read(Dex.Section in, IndexMap indexMap, int index) { - return in.readString(); + @Override String read(Dex.Section in, IndexMap indexMap, int index, int dex) { + String s = in.readString(); + if (dexes[dex].getMethodFilter().checkStringId(index) == MethodFilter.Usage.WHITELISTED) { + if (dex == 1) { + Log.w("Whitelisted: " + index + " - " + s); + } + return s; + } else { + return null; + } } @Override void updateIndex(int offset, IndexMap indexMap, int oldIndex, int newIndex) { - indexMap.stringIds[oldIndex] = newIndex; + if (newIndex == INDEX_BLACKLISTED) { + // blacklisted + indexMap.stringIds[oldIndex] = -1; + } else { + indexMap.stringIds[oldIndex] = newIndex; + } } @Override void write(String value) { @@ -465,21 +516,30 @@ private void mergeStringIds() { } private void mergeTypeIds() { + Log.d("DexMerger","mergeTypeIds..."); new IdMerger(idsDefsOut) { @Override TableOfContents.Section getSection(TableOfContents tableOfContents) { return tableOfContents.typeIds; } - @Override Integer read(Dex.Section in, IndexMap indexMap, int index) { + @Override Integer read(Dex.Section in, IndexMap indexMap, int index, int dex) { int stringIndex = in.readInt(); - return indexMap.adjustString(stringIndex); + if (dexes[dex].getMethodFilter().checkTypeId((short) index) == MethodFilter.Usage.WHITELISTED) { + return indexMap.adjustString(stringIndex); + } else { + return null; + } } @Override void updateIndex(int offset, IndexMap indexMap, int oldIndex, int newIndex) { - if (newIndex < 0 || newIndex > 0xffff) { - throw new DexIndexOverflowException("type ID not in [0, 0xffff]: " + newIndex); + if (newIndex == INDEX_BLACKLISTED) { + // blacklisted + indexMap.typeIds[oldIndex] = null; + } else if (newIndex < 0 || newIndex > 0xffff) { + throw new DexIndexOverflowException("Too many type IDs. Type ID not in [0, 0xffff]: " + newIndex); + } else { + indexMap.typeIds[oldIndex] = (short) newIndex; } - indexMap.typeIds[oldIndex] = (short) newIndex; } @Override void write(Integer value) { @@ -489,6 +549,7 @@ private void mergeTypeIds() { } private void mergeTypeLists() { + Log.d("DexMerger","mergeTypeLists..."); new IdMerger(typeListOut) { @Override TableOfContents.Section getSection(TableOfContents tableOfContents) { return tableOfContents.typeLists; @@ -509,20 +570,30 @@ private void mergeTypeLists() { } private void mergeProtoIds() { + Log.d("DexMerger","mergeProtoIds..."); new IdMerger(idsDefsOut) { @Override TableOfContents.Section getSection(TableOfContents tableOfContents) { return tableOfContents.protoIds; } - @Override ProtoId read(Dex.Section in, IndexMap indexMap, int index) { - return indexMap.adjust(in.readProtoId()); + @Override ProtoId read(Dex.Section in, IndexMap indexMap, int index, int dex) { + ProtoId m = in.readProtoId(); + if (dexes[dex].getMethodFilter().checkProtoId((short) index) == MethodFilter.Usage.WHITELISTED) { + return indexMap.adjust(m); + } else { + return null; + } } @Override void updateIndex(int offset, IndexMap indexMap, int oldIndex, int newIndex) { - if (newIndex < 0 || newIndex > 0xffff) { - throw new DexIndexOverflowException("proto ID not in [0, 0xffff]: " + newIndex); + if (newIndex == INDEX_BLACKLISTED) { + // blacklisted + indexMap.protoIds[oldIndex] = null; + } else if (newIndex < 0 || newIndex > 0xffff) { + throw new DexIndexOverflowException("Too many proto IDs. Proto ID not in [0, 0xffff]: " + newIndex); + } else { + indexMap.protoIds[oldIndex] = (short) newIndex; } - indexMap.protoIds[oldIndex] = (short) newIndex; } @Override void write(ProtoId value) { @@ -532,20 +603,30 @@ private void mergeProtoIds() { } private void mergeFieldIds() { + Log.d("DexMerger","mergeFieldIds..."); new IdMerger(idsDefsOut) { @Override TableOfContents.Section getSection(TableOfContents tableOfContents) { return tableOfContents.fieldIds; } - @Override FieldId read(Dex.Section in, IndexMap indexMap, int index) { - return indexMap.adjust(in.readFieldId()); + @Override FieldId read(Dex.Section in, IndexMap indexMap, int index, int dex) { + FieldId m = in.readFieldId(); + if (dexes[dex].getMethodFilter().checkFieldId((short) index) == MethodFilter.Usage.WHITELISTED) { + return indexMap.adjust(m); + } else { + return null; + } } @Override void updateIndex(int offset, IndexMap indexMap, int oldIndex, int newIndex) { - if (newIndex < 0 || newIndex > 0xffff) { - throw new DexIndexOverflowException("field ID not in [0, 0xffff]: " + newIndex); + if (newIndex == INDEX_BLACKLISTED) { + // blacklisted + indexMap.fieldIds[oldIndex] = null; + } else if (newIndex < 0 || newIndex > 0xffff) { + throw new DexIndexOverflowException("Too many field IDs. field ID not in [0, 0xffff]: " + newIndex); + } else { + indexMap.fieldIds[oldIndex] = (short) newIndex; } - indexMap.fieldIds[oldIndex] = (short) newIndex; } @Override void write(FieldId value) { @@ -555,41 +636,64 @@ private void mergeFieldIds() { } private void mergeMethodIds() { + Log.d("DexMerger","mergeMethodIds..."); new IdMerger(idsDefsOut) { @Override TableOfContents.Section getSection(TableOfContents tableOfContents) { return tableOfContents.methodIds; } - @Override MethodId read(Dex.Section in, IndexMap indexMap, int index) { - return indexMap.adjust(in.readMethodId()); + @Override MethodId read(Dex.Section in, IndexMap indexMap, int index, int dex) { + MethodId m = in.readMethodId(); + if (dexes[dex].getMethodFilter().checkMethodId((short) index) == MethodFilter.Usage.WHITELISTED) { + return indexMap.adjust(m); + } else { + return null; + } } @Override void updateIndex(int offset, IndexMap indexMap, int oldIndex, int newIndex) { - if (newIndex < 0 || newIndex > 0xffff) { + if (newIndex == INDEX_BLACKLISTED) { + // blacklisted + indexMap.methodIds[oldIndex] = null; + } else if (newIndex < 0 || newIndex > 0xffff) { throw new DexIndexOverflowException( - "method ID not in [0, 0xffff]: " + newIndex); + "Too many method IDs. method ID not in [0, 0xffff]: " + newIndex); + } else { + indexMap.methodIds[oldIndex] = (short) newIndex; } - indexMap.methodIds[oldIndex] = (short) newIndex; } @Override void write(MethodId methodId) { methodId.writeTo(idsDefsOut); } }.mergeSorted(); + } private void mergeAnnotations() { + Log.d("DexMerger","mergeAnnotations..."); new IdMerger(annotationOut) { @Override TableOfContents.Section getSection(TableOfContents tableOfContents) { return tableOfContents.annotations; } @Override Annotation read(Dex.Section in, IndexMap indexMap, int index) { - return indexMap.adjust(in.readAnnotation()); + Annotation annotation = in.readAnnotation(); + EncodedValueReader reader = annotation.getReader(); + int fieldCount = reader.readAnnotation(); + int type = indexMap.adjustType(reader.getAnnotationType()); + for (int i = 0; i < fieldCount; i++) { + reader.readAnnotationName(); + } + if (type != INDEX_BLACKLISTED) { + return indexMap.adjust(annotation); + } else { + return null; + } } @Override void updateIndex(int offset, IndexMap indexMap, int oldIndex, int newIndex) { - indexMap.putAnnotationOffset(offset, annotationOut.getPosition()); + indexMap.putAnnotationOffset(offset, newIndex != INDEX_BLACKLISTED ? annotationOut.getPosition() : 0); } @Override void write(Annotation value) { @@ -599,6 +703,7 @@ private void mergeAnnotations() { } private void mergeClassDefs() { + Log.d("DexMerger","mergeClassDefs..."); SortableType[] types = getSortedTypes(); contentsOut.classDefs.off = idsDefsOut.getPosition(); contentsOut.classDefs.size = types.length; @@ -610,6 +715,7 @@ private void mergeClassDefs() { } private void mergeMainDexClassDefs() { + Log.d("DexMerger","mergeMainClassDefs..."); SortableType[] types = getMainDexSortedTypes(); contentsOut.classDefs.off = idsDefsOut.getPosition(); contentsOut.classDefs.size = types.length; @@ -820,47 +926,72 @@ private void transformClassDef(Dex in, ClassDef classDef, IndexMap indexMap) { */ private void transformAnnotationDirectory( Dex.Section directoryIn, IndexMap indexMap) { - contentsOut.annotationsDirectories.size++; - annotationsDirectoryOut.assertFourByteAligned(); - indexMap.putAnnotationDirectoryOffset( - directoryIn.getPosition(), annotationsDirectoryOut.getPosition()); + List writeBuffer = new LinkedList<>(); + int directoryOffset = directoryIn.getPosition(); int classAnnotationsOffset = indexMap.adjustAnnotationSet(directoryIn.readInt()); - annotationsDirectoryOut.writeInt(classAnnotationsOffset); int fieldsSize = directoryIn.readInt(); - annotationsDirectoryOut.writeInt(fieldsSize); + int newFieldsSize = 0; int methodsSize = directoryIn.readInt(); - annotationsDirectoryOut.writeInt(methodsSize); + int newMethodsSize = 0; int parameterListSize = directoryIn.readInt(); - annotationsDirectoryOut.writeInt(parameterListSize); + int newParameterListSize = 0; for (int i = 0; i < fieldsSize; i++) { // field index - annotationsDirectoryOut.writeInt(indexMap.adjustField(directoryIn.readInt())); - - // annotations offset - annotationsDirectoryOut.writeInt(indexMap.adjustAnnotationSet(directoryIn.readInt())); + int fieldIndex = indexMap.adjustField(directoryIn.readInt()); + int offset = indexMap.adjustAnnotationSet(directoryIn.readInt()); + if (fieldIndex != INDEX_BLACKLISTED && offset > 0) { + newFieldsSize++; + // not blacklisted + writeBuffer.add(fieldIndex); + + writeBuffer.add(offset); + } } for (int i = 0; i < methodsSize; i++) { // method index - annotationsDirectoryOut.writeInt(indexMap.adjustMethod(directoryIn.readInt())); - - // annotation set offset - annotationsDirectoryOut.writeInt( - indexMap.adjustAnnotationSet(directoryIn.readInt())); + int methodIndex = indexMap.adjustMethod(directoryIn.readInt()); + int offset = indexMap.adjustAnnotationSet(directoryIn.readInt()); + if (methodIndex != INDEX_BLACKLISTED && offset > 0) { + newMethodsSize++; + // not blacklisted + writeBuffer.add(methodIndex); + + writeBuffer.add(offset); + } } for (int i = 0; i < parameterListSize; i++) { // method index - annotationsDirectoryOut.writeInt(indexMap.adjustMethod(directoryIn.readInt())); - - // annotations offset - annotationsDirectoryOut.writeInt( - indexMap.adjustAnnotationSetRefList(directoryIn.readInt())); + int methodIndex = indexMap.adjustMethod(directoryIn.readInt()); + int offset = indexMap.adjustAnnotationSetRefList(directoryIn.readInt()); + if (methodIndex != INDEX_BLACKLISTED && offset > 0) { + newParameterListSize++; + // not blacklisted + writeBuffer.add(methodIndex); + + writeBuffer.add(offset); + } + } + if (classAnnotationsOffset != 0 || newFieldsSize+newMethodsSize+newParameterListSize > 0) { + contentsOut.annotationsDirectories.size++; + annotationsDirectoryOut.assertFourByteAligned(); + indexMap.putAnnotationDirectoryOffset( + directoryOffset, annotationsDirectoryOut.getPosition()); + annotationsDirectoryOut.writeInt(classAnnotationsOffset); + annotationsDirectoryOut.writeInt(newFieldsSize); + annotationsDirectoryOut.writeInt(newMethodsSize); + annotationsDirectoryOut.writeInt(newParameterListSize); + for (Integer value : writeBuffer) { + annotationsDirectoryOut.writeInt(value); + } + } else { + indexMap.putAnnotationDirectoryOffset(directoryOffset, 0); } } @@ -868,15 +999,31 @@ private void transformAnnotationDirectory( * Transform all annotations on a single type, member or parameter. */ private void transformAnnotationSet(IndexMap indexMap, Dex.Section setIn) { - contentsOut.annotationSets.size++; - annotationSetOut.assertFourByteAligned(); - indexMap.putAnnotationSetOffset(setIn.getPosition(), annotationSetOut.getPosition()); + int annotationSetOffset = setIn.getPosition(); int size = setIn.readInt(); - annotationSetOut.writeInt(size); + int newSize = 0; + + List out = new ArrayList<>(); for (int j = 0; j < size; j++) { - annotationSetOut.writeInt(indexMap.adjustAnnotation(setIn.readInt())); + int offset = indexMap.adjustAnnotation(setIn.readInt()); + if (offset > 0) { + out.add(offset); + newSize++; + } + } + + if (newSize > 0) { + contentsOut.annotationSets.size++; + annotationSetOut.assertFourByteAligned(); + indexMap.putAnnotationSetOffset(annotationSetOffset, annotationSetOut.getPosition()); + annotationSetOut.writeInt(newSize); + for (Integer i : out) { + annotationSetOut.writeInt(i); + } + } else { + indexMap.putAnnotationSetOffset(annotationSetOffset, 0); } } @@ -884,15 +1031,35 @@ private void transformAnnotationSet(IndexMap indexMap, Dex.Section setIn) { * Transform all annotation set ref lists. */ private void transformAnnotationSetRefList(IndexMap indexMap, Dex.Section refListIn) { - contentsOut.annotationSetRefLists.size++; - annotationSetRefListOut.assertFourByteAligned(); - indexMap.putAnnotationSetRefListOffset( - refListIn.getPosition(), annotationSetRefListOut.getPosition()); + + int annotationSetRefListOffset = refListIn.getPosition(); int parameterCount = refListIn.readInt(); - annotationSetRefListOut.writeInt(parameterCount); + int newParameterCount = 0; + + List out = new ArrayList<>(); + for (int p = 0; p < parameterCount; p++) { - annotationSetRefListOut.writeInt(indexMap.adjustAnnotationSet(refListIn.readInt())); + int offset = indexMap.adjustAnnotationSet(refListIn.readInt()); + if (offset > 0) { + out.add(offset); + newParameterCount++; + } + } + + if (newParameterCount > 0) { + contentsOut.annotationSetRefLists.size++; + annotationSetRefListOut.assertFourByteAligned(); + indexMap.putAnnotationSetRefListOffset(annotationSetRefListOffset + , annotationSetRefListOut.getPosition()); + annotationSetRefListOut.writeInt(newParameterCount); + + for (Integer i : out) { + annotationSetRefListOut.writeInt(i); + } + } else { + indexMap.putAnnotationSetRefListOffset( + annotationSetRefListOffset, 0); } } @@ -919,9 +1086,13 @@ private void transformFields(IndexMap indexMap, ClassData.Field[] fields) { int lastOutFieldIndex = 0; for (ClassData.Field field : fields) { int outFieldIndex = indexMap.adjustField(field.getFieldIndex()); - classDataOut.writeUleb128(outFieldIndex - lastOutFieldIndex); - lastOutFieldIndex = outFieldIndex; - classDataOut.writeUleb128(field.getAccessFlags()); + if (outFieldIndex != -1) { + classDataOut.writeUleb128(outFieldIndex - lastOutFieldIndex); + lastOutFieldIndex = outFieldIndex; + classDataOut.writeUleb128(field.getAccessFlags()); + } else { + throw new AssertionError("Blacklisted field " + field.toString() + " is already defined in dex "); + } } } @@ -929,17 +1100,22 @@ private void transformMethods(Dex in, IndexMap indexMap, ClassData.Method[] meth int lastOutMethodIndex = 0; for (ClassData.Method method : methods) { int outMethodIndex = indexMap.adjustMethod(method.getMethodIndex()); - classDataOut.writeUleb128(outMethodIndex - lastOutMethodIndex); - lastOutMethodIndex = outMethodIndex; - - classDataOut.writeUleb128(method.getAccessFlags()); - - if (method.getCodeOffset() == 0) { - classDataOut.writeUleb128(0); + if (outMethodIndex != INDEX_BLACKLISTED) { + // not blacklisted + classDataOut.writeUleb128(outMethodIndex - lastOutMethodIndex); + lastOutMethodIndex = outMethodIndex; + + classDataOut.writeUleb128(method.getAccessFlags()); + + if (method.getCodeOffset() == 0) { + classDataOut.writeUleb128(0); + } else { + codeOut.alignToFourBytesWithZeroFill(); + classDataOut.writeUleb128(codeOut.getPosition()); + transformCode(in, in.readCode(method), indexMap); + } } else { - codeOut.alignToFourBytesWithZeroFill(); - classDataOut.writeUleb128(codeOut.getPosition()); - transformCode(in, in.readCode(method), indexMap); + throw new AssertionError("Blacklisted method " + method.toString() + "is already defined in dex"); } } } @@ -1238,6 +1414,20 @@ public int size() { } } + public static class MergeException extends Exception { + + private final Throwable exception; + + public MergeException(Throwable exception) { + this.exception = exception; + } + + public Throwable getValue(){ + return exception; + } + + } + // public static void main(String[] args) throws IOException { // if (args.length < 2) { // printUsage(); diff --git a/src/main/java/comm/android/dx/merge/IndexMap.java b/src/main/java/comm/android/dx/merge/IndexMap.java index b292a00..190b570 100644 --- a/src/main/java/comm/android/dx/merge/IndexMap.java +++ b/src/main/java/comm/android/dx/merge/IndexMap.java @@ -39,6 +39,8 @@ import static comm.android.dex.EncodedValueReader.ENCODED_SHORT; import static comm.android.dex.EncodedValueReader.ENCODED_STRING; import static comm.android.dex.EncodedValueReader.ENCODED_TYPE; +import static comm.android.dx.merge.DexMerger.INDEX_BLACKLISTED; + import comm.android.dex.EncodedValueCodec; import comm.android.dex.FieldId; import comm.android.dex.Leb128; @@ -62,10 +64,10 @@ public final class IndexMap { private final Dex target; public final int[] stringIds; - public final short[] typeIds; - public final short[] protoIds; - public final short[] fieldIds; - public final short[] methodIds; + public final Short[] typeIds; + public final Short[] protoIds; + public final Short[] fieldIds; + public final Short[] methodIds; private final HashMap typeListOffsets; private final HashMap annotationOffsets; private final HashMap annotationSetOffsets; @@ -76,10 +78,10 @@ public final class IndexMap { public IndexMap(Dex target, TableOfContents tableOfContents) { this.target = target; this.stringIds = new int[tableOfContents.stringIds.size]; - this.typeIds = new short[tableOfContents.typeIds.size]; - this.protoIds = new short[tableOfContents.protoIds.size]; - this.fieldIds = new short[tableOfContents.fieldIds.size]; - this.methodIds = new short[tableOfContents.methodIds.size]; + this.typeIds = new Short[tableOfContents.typeIds.size]; + this.protoIds = new Short[tableOfContents.protoIds.size]; + this.fieldIds = new Short[tableOfContents.fieldIds.size]; + this.methodIds = new Short[tableOfContents.methodIds.size]; this.typeListOffsets = new HashMap(); this.annotationOffsets = new HashMap(); this.annotationSetOffsets = new HashMap(); @@ -105,28 +107,28 @@ public void putTypeListOffset(int oldOffset, int newOffset) { } public void putAnnotationOffset(int oldOffset, int newOffset) { - if (oldOffset <= 0 || newOffset <= 0) { + if (oldOffset <= 0 || newOffset < 0) { throw new IllegalArgumentException(); } annotationOffsets.put(oldOffset, newOffset); } public void putAnnotationSetOffset(int oldOffset, int newOffset) { - if (oldOffset <= 0 || newOffset <= 0) { + if (oldOffset <= 0 || newOffset < 0) { throw new IllegalArgumentException(); } annotationSetOffsets.put(oldOffset, newOffset); } public void putAnnotationSetRefListOffset(int oldOffset, int newOffset) { - if (oldOffset <= 0 || newOffset <= 0) { + if (oldOffset <= 0 || newOffset < 0) { throw new IllegalArgumentException(); } annotationSetRefListOffsets.put(oldOffset, newOffset); } public void putAnnotationDirectoryOffset(int oldOffset, int newOffset) { - if (oldOffset <= 0 || newOffset <= 0) { + if (oldOffset <= 0 || newOffset < 0) { throw new IllegalArgumentException(); } annotationDirectoryOffsets.put(oldOffset, newOffset); @@ -140,11 +142,14 @@ public void putStaticValuesOffset(int oldOffset, int newOffset) { } public int adjustString(int stringIndex) { + if (stringIndex != ClassDef.NO_INDEX && stringIds[stringIndex] == INDEX_BLACKLISTED) { + return 0; + } return stringIndex == ClassDef.NO_INDEX ? ClassDef.NO_INDEX : stringIds[stringIndex]; } public int adjustType(int typeIndex) { - return (typeIndex == ClassDef.NO_INDEX) ? ClassDef.NO_INDEX : (typeIds[typeIndex] & 0xffff); + return (typeIndex == ClassDef.NO_INDEX || typeIds[typeIndex] == null ) ? ClassDef.NO_INDEX : (typeIds[typeIndex] & 0xffff); } public TypeList adjustTypeList(TypeList typeList) { @@ -159,15 +164,15 @@ public TypeList adjustTypeList(TypeList typeList) { } public int adjustProto(int protoIndex) { - return protoIds[protoIndex] & 0xffff; + return protoIds[protoIndex]==null ? ClassDef.NO_INDEX:protoIds[protoIndex] & 0xffff; } public int adjustField(int fieldIndex) { - return fieldIds[fieldIndex] & 0xffff; + return fieldIds[fieldIndex]==null ? ClassDef.NO_INDEX:fieldIds[fieldIndex] & 0xffff; } public int adjustMethod(int methodIndex) { - return methodIds[methodIndex] & 0xffff; + return methodIds[methodIndex]==null ? ClassDef.NO_INDEX: (methodIds[methodIndex] & 0xffff); } public int adjustTypeListOffset(int typeListOffset) { @@ -253,7 +258,7 @@ public Annotation adjust(Annotation annotation) { /** * Adjust an encoded value or array. */ - private final class EncodedValueTransformer { + public final class EncodedValueTransformer { private final ByteOutput out; public EncodedValueTransformer(ByteOutput out) { @@ -292,20 +297,23 @@ public void transform(EncodedValueReader reader) { out, ENCODED_STRING, adjustString(reader.readString())); break; case ENCODED_TYPE: + int type = adjustType(reader.readType()); EncodedValueCodec.writeUnsignedIntegralValue( - out, ENCODED_TYPE, adjustType(reader.readType())); + out, ENCODED_TYPE, (type >= 0 && type <= 0xffff) ? type : 0); break; case ENCODED_FIELD: + int field = adjustField(reader.readField()); EncodedValueCodec.writeUnsignedIntegralValue( - out, ENCODED_FIELD, adjustField(reader.readField())); + out, ENCODED_FIELD, (field >= 0 && field <= 0xffff) ? field : 0); break; case ENCODED_ENUM: EncodedValueCodec.writeUnsignedIntegralValue( out, ENCODED_ENUM, adjustField(reader.readEnum())); break; case ENCODED_METHOD: + int methodid = adjustMethod(reader.readMethod()); EncodedValueCodec.writeUnsignedIntegralValue( - out, ENCODED_METHOD, adjustMethod(reader.readMethod())); + out, ENCODED_METHOD, (methodid >= 0 && methodid <= 0xffff) ? methodid : 0); break; case ENCODED_ARRAY: writeTypeAndArg(ENCODED_ARRAY, 0); @@ -330,7 +338,8 @@ public void transform(EncodedValueReader reader) { private void transformAnnotation(EncodedValueReader reader) { int fieldCount = reader.readAnnotation(); - Leb128.writeUnsignedLeb128(out, adjustType(reader.getAnnotationType())); + int type = adjustType(reader.getAnnotationType()); + Leb128.writeUnsignedLeb128(out, type == INDEX_BLACKLISTED ? 0 : type); Leb128.writeUnsignedLeb128(out, fieldCount); for (int i = 0; i < fieldCount; i++) { Leb128.writeUnsignedLeb128(out, adjustString(reader.readAnnotationName())); diff --git a/src/main/java/comm/android/dx/merge/InstructionTransformer.java b/src/main/java/comm/android/dx/merge/InstructionTransformer.java index 68b27b5..c40be17 100644 --- a/src/main/java/comm/android/dx/merge/InstructionTransformer.java +++ b/src/main/java/comm/android/dx/merge/InstructionTransformer.java @@ -23,6 +23,8 @@ import comm.android.dx.io.instructions.DecodedInstruction; import comm.android.dx.io.instructions.ShortArrayCodeOutput; +import static comm.android.dx.merge.DexMerger.INDEX_BLACKLISTED; + final class InstructionTransformer { private final CodeReader reader; @@ -100,6 +102,9 @@ private class MethodVisitor implements CodeReader.Visitor { public void visit(DecodedInstruction[] all, DecodedInstruction one) { int methodId = one.getIndex(); int mappedId = indexMap.adjustMethod(methodId); + if (mappedId == INDEX_BLACKLISTED) { + throw new IllegalArgumentException("Instruction accesses blacklisted method"); + } boolean isJumbo = (one.getOpcode() == Opcodes.CONST_STRING_JUMBO); jumboCheck(isJumbo, mappedId); mappedInstructions[mappedAt++] = one.withIndex(mappedId); diff --git a/src/main/java/comm/android/dx/merge/MethodFilter.java b/src/main/java/comm/android/dx/merge/MethodFilter.java new file mode 100644 index 0000000..3fab73a --- /dev/null +++ b/src/main/java/comm/android/dx/merge/MethodFilter.java @@ -0,0 +1,408 @@ +package comm.android.dx.merge; + + +import java.util.HashMap; +import java.util.LinkedHashMap; + +import comm.android.dex.Annotation; +import comm.android.dex.Dex; +import comm.android.dex.EncodedValueReader; +import comm.android.dex.FieldId; +import comm.android.dex.MethodId; +import comm.android.dex.ProtoId; +import comm.android.dex.TableOfContents; +import trikita.log.Log; + + +public class MethodFilter { + + private final String annotationType; + protected final Dex dex; + + private final boolean skip; + + public enum Usage { + BLACKLISTED, + STRIP, + WHITELISTED; + } + + protected HashMap methodIdWhitelist; + protected HashMap fieldIdWhitelist; + protected HashMap protoIdWhitelist; + protected HashMap typeIdWhitelist; + protected HashMap stringIdWhitelist; + private static final String TAG = "MethodFilter"; + + + private Usage check(HashMap whitelist, short id) { + Usage usage = whitelist.get(id); + return usage!=null?usage:Usage.BLACKLISTED; + } + + private Usage check(HashMap whitelist, int id) { + Usage usage = whitelist.get(id); + return usage!=null?usage:Usage.BLACKLISTED; + } + + Usage checkMethodId(short methodId){ + return skip?Usage.WHITELISTED:check(methodIdWhitelist, methodId); + } + + Usage checkFieldId(short fieldId){ + return skip?Usage.WHITELISTED:check(fieldIdWhitelist, fieldId); + } + + Usage checkProtoId(short protoId){ + return skip?Usage.WHITELISTED:check(protoIdWhitelist, protoId); + } + + Usage checkTypeId(short typeId){ + return skip?Usage.WHITELISTED:check(typeIdWhitelist, typeId); + } + + Usage checkStringId(int stringId){ + return skip?Usage.WHITELISTED:check(stringIdWhitelist, stringId); + } + + void reset(){ + methodIdWhitelist = new LinkedHashMap<>(); + fieldIdWhitelist = new LinkedHashMap<>(); + protoIdWhitelist = new LinkedHashMap<>(); + typeIdWhitelist = new LinkedHashMap<>(); + stringIdWhitelist = new LinkedHashMap<>(); + } + + private void processMethodIds(){ + TableOfContents.Section section = dex.getTableOfContents().methodIds; + Dex.Section in = dex.open(section.off); + for (short j = 0; j < section.size; j++) { + MethodId m = in.readMethodId(); + if (checkMethodId(j) == Usage.WHITELISTED) { + methodIdWhitelist.put((short) j, Usage.WHITELISTED); + typeIdWhitelist.put((short) m.getDeclaringClassIndex(), Usage.WHITELISTED); + stringIdWhitelist.put(m.getNameIndex(), Usage.WHITELISTED); + protoIdWhitelist.put((short) m.getProtoIndex(), Usage.WHITELISTED); + } + } + } + + private void processFieldIds(){ + TableOfContents.Section fieldIds = dex.getTableOfContents().fieldIds; + Dex.Section section = dex.open(fieldIds.off); + for (short j = 0; j < fieldIds.size; j++) { + FieldId m = section.readFieldId(); + if (checkFieldId(j) == Usage.WHITELISTED) { + typeIdWhitelist.put((short) m.getDeclaringClassIndex(), Usage.WHITELISTED); + stringIdWhitelist.put(m.getNameIndex(), Usage.WHITELISTED); + protoIdWhitelist.put((short) m.getTypeIndex(), Usage.WHITELISTED); + } + } + } + + private void processProtoIds(){ + TableOfContents.Section protoIds = dex.getTableOfContents().protoIds; + Dex.Section section = dex.open(protoIds.off); + for (int j = 0; j < protoIds.size; j++) { + ProtoId m = section.readProtoId(); + if (checkProtoId((short) j) == Usage.WHITELISTED) { + stringIdWhitelist.put(m.getShortyIndex(), Usage.WHITELISTED); + typeIdWhitelist.put((short) m.getReturnTypeIndex(), Usage.WHITELISTED); + int parametersOffset = m.getParametersOffset(); + if (parametersOffset != 0) { + Dex.Section in2 = dex.open(m.getParametersOffset()); + int size = in2.readInt(); + for (short k = 0; k < size; k++){ + typeIdWhitelist.put(in2.readShort(), Usage.WHITELISTED); + } + } + } + } + } + + private void processTypeIds(){ + TableOfContents.Section typeIds = dex.getTableOfContents().typeIds; + Dex.Section section = dex.open(typeIds.off); + for (int j = 0; j < typeIds.size; j++) { + int m = section.readInt(); + if (checkTypeId((short) j) == Usage.WHITELISTED) { + stringIdWhitelist.put(m, Usage.WHITELISTED); + } + } + } + + + protected String getString(int stringid){ + Dex.Section stringsSection = dex.open(dex.getTableOfContents().stringIds.off+4*stringid); + return stringsSection.readString(); + } + + + protected String getTypeString(int typeid){ + Dex.Section typesSection = dex.open(dex.getTableOfContents().typeIds.off+4*typeid); + int stringid = typesSection.readInt(); + return getString(stringid); + } + + protected String getMethodString(int methodid){ + Dex.Section typesSection = dex.open(dex.getTableOfContents().typeIds.off+8*methodid+4); + int stringid = typesSection.readInt(); + return getString(stringid); + } + + protected void initializeWhitelist() throws DexMerger.MergeException { + if (!skip) { + + int annotationtypeid = -1; + + TableOfContents.Section typeIds = dex.getTableOfContents().typeIds; + Dex.Section section = dex.open(typeIds.off); + for (int j = 0; j < typeIds.size; j++) { + int m = section.readInt(); + if (getString(m).equals(annotationType)) { + annotationtypeid = j; + } + } + + Log.i(TAG, "Annotationtype used for whitelisting:" + annotationType); + + if (annotationtypeid == -1) { + + throw new DexMerger.MergeException(new IllegalArgumentException("Annotation type " + annotationType + " is not defined in the codelib dex file")); + + } else { + + typeIdWhitelist.put((short)annotationtypeid, Usage.STRIP); + + } + + int classdefs = dex.getTableOfContents().classDefs.size; + Dex.Section classDefsSection = dex.open(dex.getTableOfContents().classDefs.off); + for (int classdefsi = 0; classdefsi < classdefs; classdefsi ++) { + int class_idx = classDefsSection.readInt(); + classDefsSection.readInt(); // access_flags + classDefsSection.readInt(); // superclass_idx + classDefsSection.readInt(); // interfaces_off + classDefsSection.readInt(); // source_file_idx + int annotations_directory_off = classDefsSection.readInt(); + classDefsSection.readInt(); // class_data_off + classDefsSection.readInt(); // static_values_off + if (annotations_directory_off != 0) { + Dex.Section annotations_directory = dex.open(annotations_directory_off); + /* + class_annotations_off: uint + fields_size: uint + methods_dize: uint + parameters_size: uint + field_annotations: field_annotation[fields_size] + method_annotations: method_annotation[methods_size] + parameter_annotations: parameter_annotation[parameters_size] + */ + int class_annotations_off = annotations_directory.readInt(); + int fields_size = annotations_directory.readInt(); + int methods_size = annotations_directory.readInt(); + int parameters_size = annotations_directory.readInt(); + + for (int k = 0; k < fields_size; k++) { + int fieldid = annotations_directory.readInt(); + int annotations_off = annotations_directory.readInt(); + Dex.Section annotation_set_item = dex.open(annotations_off); + int size = annotation_set_item.readInt(); + for (int l = 0; l < size; l++) { + Dex.Section annotation_off_item = dex.open(annotation_set_item.readInt()); + Annotation annotation = annotation_off_item.readAnnotation(); + if (annotation.getTypeIndex() == annotationtypeid) { + Log.d(TAG, "Field whitelisted:" + fieldid); + typeIdWhitelist.put((short) class_idx, Usage.WHITELISTED); + fieldIdWhitelist.put((short) fieldid, Usage.WHITELISTED); + } + } + } + for (int k = 0; k < methods_size; k++) { + int methodid = annotations_directory.readInt(); + int annotations_off = annotations_directory.readInt(); + Dex.Section annotation_set_item = dex.open(annotations_off); + int size = annotation_set_item.readInt(); + for (int l = 0; l < size; l++) { + Dex.Section annotation_off_item = dex.open(annotation_set_item.readInt()); + Annotation annotation = annotation_off_item.readAnnotation(); + if (annotation.getTypeIndex() == annotationtypeid) { + Log.d("ColdelibWhitelisting", "Method annotated:" + getMethodString(methodid)); + typeIdWhitelist.put((short) class_idx, Usage.WHITELISTED); + methodIdWhitelist.put((short) methodid, Usage.WHITELISTED); + } + } + } + for (int k = 0; k < parameters_size; k++) { + annotations_directory.readInt(); // methodid + annotations_directory.readInt(); // offset + } + if (class_annotations_off != 0) { + Dex.Section class_annotation = dex.open(class_annotations_off); + int class_annotation_size = class_annotation.readInt(); + for (int class_annotation_index = 0; class_annotation_index < class_annotation_size; class_annotation_index++) { + Dex.Section class_annotation_item = dex.open(class_annotation.readInt()); + Annotation annotation = class_annotation_item.readAnnotation(); + if (annotation.getTypeIndex() == annotationtypeid) { + typeIdWhitelist.put((short) class_idx, Usage.WHITELISTED); + } + } + } + if (typeIdWhitelist.get((short) class_idx) == Usage.WHITELISTED) + Log.i("CodelibWhitelisting", "Class whitelisted: " + String.valueOf(class_idx) + " (" + getTypeString(class_idx) + + ")\nclass_annotations_off: " + class_annotations_off + "\nf/m/p:" + fields_size + "/" + methods_size + "/" + parameters_size); + } + } + } + } + + + + + private void processAnnotations(){ + Dex.Section classDefsSection = dex.open(dex.getTableOfContents().classDefs.off); + int classdefs = dex.getTableOfContents().classDefs.size; + for (int classdefsi = 0; classdefsi < classdefs; classdefsi ++) { + int class_idx = classDefsSection.readInt(); + classDefsSection.readInt(); // access_flags + classDefsSection.readInt(); // superclass_idx + classDefsSection.readInt(); // interfaces_off + classDefsSection.readInt(); // source_file_idx + int annotations_directory_off = classDefsSection.readInt(); + classDefsSection.readInt(); // class_data_off + classDefsSection.readInt(); // static_values_off + if (annotations_directory_off != 0 && checkTypeId((short) class_idx) == Usage.WHITELISTED) { + Dex.Section annotations_directory = dex.open(annotations_directory_off); + /* + class_annotations_off: uint + fields_size: uint + methods_dize: uint + parameters_size: uint + field_annotations: field_annotation[fields_size] + method_annotations: method_annotation[methods_size] + parameter_annotations: parameter_annotation[parameters_size] + */ + int class_annotations_off = annotations_directory.readInt(); + if (class_annotations_off != 0) { + Dex.Section class_annotation = dex.open(class_annotations_off); + int class_annotation_size = class_annotation.readInt(); + for (int class_annotation_index = 0; class_annotation_index < class_annotation_size; class_annotation_index++) { + Dex.Section class_annotation_item = dex.open(class_annotation.readInt()); + Annotation annotation = class_annotation_item.readAnnotation(); + if (checkTypeId((short) annotation.getTypeIndex()) != Usage.STRIP) { + typeIdWhitelist.put((short) annotation.getTypeIndex(), Usage.WHITELISTED); + EncodedValueReader encodedValueReader = annotation.getReader(); + int annotation_size = encodedValueReader.readAnnotation(); + for (int m = 0; m < annotation_size; m++) { + stringIdWhitelist.put(encodedValueReader.readAnnotationName(), Usage.WHITELISTED); + encodedValueReader.skipValue(); + } + } + } + } + int fields_size = annotations_directory.readInt(); + int methods_size = annotations_directory.readInt(); + int parameters_size = annotations_directory.readInt(); + for (int k = 0; k < fields_size; k++) { + int fieldid = annotations_directory.readInt(); + int annotations_off = annotations_directory.readInt(); + if (checkFieldId((short) fieldid) == Usage.WHITELISTED) { + Dex.Section in2 = dex.open(annotations_off); + int size = in2.readInt(); + for (int l = 0; l < size; l++) { + Dex.Section in3 = dex.open(in2.readInt()); + Annotation annotation = in3.readAnnotation(); + if (checkTypeId((short) annotation.getTypeIndex()) != Usage.STRIP) { + typeIdWhitelist.put((short) annotation.getTypeIndex(), Usage.WHITELISTED); + EncodedValueReader r = annotation.getReader(); + int annotation_size = r.readAnnotation(); + for (int m = 0; m < annotation_size; m++) { + stringIdWhitelist.put(r.readAnnotationName(), Usage.WHITELISTED); + r.skipValue(); + } + } + } + } + } + for (int k = 0; k < methods_size; k++) { + int methodid = annotations_directory.readInt(); + int annotations_off = annotations_directory.readInt(); + if (checkMethodId((short) methodid) == Usage.WHITELISTED) { + Dex.Section in2 = dex.open(annotations_off); + int size = in2.readInt(); + for (int l = 0; l < size; l++) { + Dex.Section in3 = dex.open(in2.readInt()); + Annotation annotation = in3.readAnnotation(); + if (checkTypeId((short) annotation.getTypeIndex()) != Usage.STRIP) { + typeIdWhitelist.put((short) annotation.getTypeIndex(), Usage.WHITELISTED); + EncodedValueReader r = annotation.getReader(); + int annotation_size = r.readAnnotation(); + for (int m = 0; m < annotation_size; m++) { + stringIdWhitelist.put(r.readAnnotationName(), Usage.WHITELISTED); + r.skipValue(); + } + } + } + } + } + for (int k = 0; k < parameters_size; k++) { + int methodid = annotations_directory.readInt(); + int annotations_off = annotations_directory.readInt(); + if (checkMethodId((short) methodid) == Usage.WHITELISTED) { + Dex.Section in2 = dex.open(annotations_off); + int size = in2.readInt(); + for (int l = 0; l < size; l++) { + int aoff = in2.readInt(); + if (aoff != 0) { + Dex.Section in3 = dex.open(aoff); + int size2 = in3.readInt(); + for (int m = 0; m < size2; m++) { + Dex.Section in4 = dex.open(in3.readInt()); + Annotation annotation = in4.readAnnotation(); + EncodedValueReader reader = annotation.getReader(); + int annotation_size = reader.readAnnotation(); + for (int o = 0; o < annotation_size; o++) { + stringIdWhitelist.put(reader.readAnnotationName(), Usage.WHITELISTED); + reader.skipValue(); + } + } + } + } + } + } + } + } + } + + + public MethodFilter(Dex dex, String annotationType) throws DexMerger.MergeException { + this.dex = dex; + this.annotationType = annotationType; + reset(); + + skip = annotationType == null; + + if (!skip) { + + initializeWhitelist(); + + for (int i = 0; i < 2; i++) { + + processMethodIds(); + + processFieldIds(); + + processProtoIds(); + + processTypeIds(); + + processAnnotations(); + } + + } + + } + +} + + diff --git a/src/main/java/saarland/cispa/apksigner/zipio/ZipInput.java b/src/main/java/saarland/cispa/apksigner/zipio/ZipInput.java index 726641a..cfd9859 100644 --- a/src/main/java/saarland/cispa/apksigner/zipio/ZipInput.java +++ b/src/main/java/saarland/cispa/apksigner/zipio/ZipInput.java @@ -204,7 +204,7 @@ public String readString(int length) throws IOException { for (int i = 0; i < length; i++) { buffer[i] = in.readByte(); } - return new String(buffer); + return new String(buffer, "ISO-8859-1"); } public byte[] readBytes(int length) throws IOException { diff --git a/src/main/java/saarland/cispa/apksigner/zipio/ZipOutput.java b/src/main/java/saarland/cispa/apksigner/zipio/ZipOutput.java index 99d131a..0fd3db2 100644 --- a/src/main/java/saarland/cispa/apksigner/zipio/ZipOutput.java +++ b/src/main/java/saarland/cispa/apksigner/zipio/ZipOutput.java @@ -125,7 +125,7 @@ public void writeShort(short value) throws IOException { public void writeString(String value) throws IOException { - byte[] data = value.getBytes(); + byte[] data = value.getBytes("ISO-8859-1"); out.write(data); filePointer += data.length; } diff --git a/src/main/java/saarland/cispa/dexterous/Dexterous.java b/src/main/java/saarland/cispa/dexterous/Dexterous.java index f7ebebd..3c76848 100644 --- a/src/main/java/saarland/cispa/dexterous/Dexterous.java +++ b/src/main/java/saarland/cispa/dexterous/Dexterous.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.Charset; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Locale; @@ -95,7 +96,7 @@ private boolean hasMultipleDexes() { return dexBuffers.size() > 1; } - private void mergeMethodIds(final String DEX_NAME, Dex dexFile) { + private void mergeMethodIds(final String DEX_NAME, Dex dexFile) throws DexMerger.MergeException { if (!hasMultipleDexes()) { Log.e(TAG, String.format("mergeMethodIds: NO Multiple DexFiles Found: Singular DexFile only.")); return; @@ -111,7 +112,8 @@ private void mergeMethodIds(final String DEX_NAME, Dex dexFile) { } } - public void mergeCodeLib() { + public void mergeCodeLib() throws DexMerger.MergeException { + dexBuffers.get(CODE_LIB_DEX_NAME).setWhitelistedAnnotation("Lsaarland/cispa/artist/codelib/CodeLib$Inject;"); for (Map.Entry dexfile : dexBuffers.entrySet()) { final String DEX_NAME = dexfile.getKey(); Dex dexFile = dexfile.getValue(); @@ -136,16 +138,16 @@ public String buildApk() { } try ( ZipInputStream zipInput = - new ZipInputStream(new BufferedInputStream(apk_original)); + new ZipInputStream(new BufferedInputStream(apk_original), Charset.forName("ISO-8859-1")); ZipOutputStream zipOutput = - new ZipOutputStream(new BufferedOutputStream(apk_injected)); + new ZipOutputStream(new BufferedOutputStream(apk_injected), Charset.forName("ISO-8859-1")); ) { ZipEntry apkContent; int classes_dex_counter = 1; while ((apkContent = zipInput.getNextEntry()) != null) { // Log.d(TAG, "> zipInput: " + apkContent.getName()); - if (apkContent.getName().contains(".dex") && classes_dex_counter == 1) { + if (apkContent.getName().endsWith(".dex") && classes_dex_counter == 1) { for (final Dex classesDex : this.dexBuffers.values()) { final String classes_dex_name; @@ -167,7 +169,7 @@ public String buildApk() { } ++classes_dex_counter; } - } else { + } else if (!apkContent.getName().endsWith(".dex")){ byte[] buffer = new byte[1024]; int count; try { @@ -184,10 +186,14 @@ public String buildApk() { apkContent.getMethod(), apkContent.getSize(), apkContent.getCompressedSize())); - zipEntry.setMethod(ZipEntry.STORED); - zipEntry.setSize(apkContent.getSize()); - // zipEntry.setCompressedSize(apkContent.getCompressedSize()); - zipEntry.setCrc(apkContent.getCrc()); + long size = apkContent.getSize(); + long crc = apkContent.getCrc(); + if (size != -1 && crc != -1) { + zipEntry.setMethod(ZipEntry.STORED); + zipEntry.setSize(size); + // zipEntry.setCompressedSize(apkContent.getCompressedSize()); + zipEntry.setCrc(crc); + } } zipOutput.putNextEntry(zipEntry); while ((count = zipInput.read(buffer)) != -1) { @@ -206,11 +212,11 @@ public String buildApk() { } } - private Dex mergeCodeLibReference(final String dexName, final Dex dexFile) { + private Dex mergeCodeLibReference(final String dexName, final Dex dexFile) throws DexMerger.MergeException { return mergeCodeLibReference(dexName, dexFile, false); } - private Dex mergeCodeLibReference(final String dexName, final Dex dexFile, final boolean saveDexFile) { + private Dex mergeCodeLibReference(final String dexName, final Dex dexFile, final boolean saveDexFile) throws DexMerger.MergeException { Dex mergedDexContent = null; dexFile.setName(dexName); @@ -225,6 +231,7 @@ private Dex mergeCodeLibReference(final String dexName, final Dex dexFile, final } catch (final IOException e) { Log.e(TAG, "", e); + mergedDexContent = dexFile; } if (saveDexFile) { diff --git a/src/main/java/saarland/cispa/dexterous/MultiDex.java b/src/main/java/saarland/cispa/dexterous/MultiDex.java index f256cd5..51a1809 100644 --- a/src/main/java/saarland/cispa/dexterous/MultiDex.java +++ b/src/main/java/saarland/cispa/dexterous/MultiDex.java @@ -63,9 +63,10 @@ public static Map loadDexfiles(final File fileContainingDex) { ZipEntry entry = zipFile.getEntry(CLASSES_DEX_FILENAME); if (entry == null) { - Log.i(TAG, String.format("ERROR Loading DexFile: %s: Not present in file: %s", - CLASSES_DEX_FILENAME, - fileContainingDex.getName())); + if (i == 1) + Log.w(TAG, String.format("ERROR Loading DexFile: %s: Not present in file: %s", + CLASSES_DEX_FILENAME, + fileContainingDex.getName())); break; } try { diff --git a/src/main/java/saarland/cispa/dexterous/cli/Dexterously.java b/src/main/java/saarland/cispa/dexterous/cli/Dexterously.java index 60dec6c..aea168d 100644 --- a/src/main/java/saarland/cispa/dexterous/cli/Dexterously.java +++ b/src/main/java/saarland/cispa/dexterous/cli/Dexterously.java @@ -279,7 +279,7 @@ private boolean hasClassDefDuplicates() { return this.classDefDuplicates.size() > 0; } - private void mergeMethodIds(final String DEX_NAME, Dex dexFile) { + private void mergeMethodIds(final String DEX_NAME, Dex dexFile) throws DexMerger.MergeException { if (!hasMultipleDexes()) { Log.e(TAG, String.format("## mergeMethodIds: NO Multiple DexFiles Found: Singular DexFile only.")); return; @@ -613,7 +613,8 @@ private void logClassesWithData(final String DEX_NAME) { ); } - public void mergeCodeLib() { + public void mergeCodeLib() throws DexMerger.MergeException { + dexBuffers.get(CODE_LIB_DEX_NAME).setWhitelistedAnnotation("Lsaarland/cispa/artist/codelib/CodeLib$Inject;"); for (Map.Entry dexfile : dexBuffers.entrySet()) { final String DEX_NAME = dexfile.getKey(); Dex dexFile = dexfile.getValue(); @@ -726,24 +727,24 @@ private void signApk(final String apkPath) { Log.e(TAG, e); } } - public Dex mergeCodeLibReference(final String dexName, final Dex dexFile) { + public Dex mergeCodeLibReference(final String dexName, final Dex dexFile) throws DexMerger.MergeException { return mergeCodeLibReference(dexName, dexFile, true); } - public Dex mergeCodeLibReference(final String dexName, final Dex dexFile, final boolean saveDexFile) { + public Dex mergeCodeLibReference(final String dexName, final Dex dexFile, final boolean saveDexFile) throws DexMerger.MergeException { Dex mergedDexContent = null; + DexMerger dexMerger = null; try { - DexMerger dexMerger = new DexMerger( + dexMerger = new DexMerger( new Dex[]{dexFile, dexBuffers.get(CODE_LIB_DEX_NAME)}, CODE_LIB_DEX_NAME, CollisionPolicy.FAIL, this.context ); - mergedDexContent = dexMerger.mergeMethodsOnly(); - - } catch (final IOException e) { - Log.e(TAG, e); + } catch (IOException e) { + throw new DexMerger.MergeException(e); } + mergedDexContent = dexMerger.mergeMethodsOnly(); if (saveDexFile) { try { diff --git a/src/main/java/saarland/cispa/dexterous/cli/Main.java b/src/main/java/saarland/cispa/dexterous/cli/Main.java index 842072f..def0800 100644 --- a/src/main/java/saarland/cispa/dexterous/cli/Main.java +++ b/src/main/java/saarland/cispa/dexterous/cli/Main.java @@ -21,6 +21,8 @@ package saarland.cispa.dexterous.cli; import org.apache.commons.cli.*; + +import comm.android.dx.merge.DexMerger; import saarland.cispa.dexterous.Config; import trikita.log.Log; @@ -42,15 +44,19 @@ public static void main(final String[] args) { } else { dexterously.info(); } - if (runConfig.build_apk) { - dexterously.mergeCodeLib(); - dexterously.buildApk(); - if (runConfig.sign_apk) { - dexterously.signApk(); + try { + if (runConfig.build_apk) { + dexterously.mergeCodeLib(); + dexterously.buildApk(); + if (runConfig.sign_apk) { + dexterously.signApk(); + } } - } - if (runConfig.merge_dex) { - dexterously.mergeDexfiles(); + if (runConfig.merge_dex) { + dexterously.mergeDexfiles(); + } + } catch (DexMerger.MergeException e) { + e.printStackTrace(); } }