GT-2754, 2756 - Decompiler Reference Finder - fixed

NullPointerException; updated the finder to handle 'anonymous field
accesses'

Ghidra/Features/DecompilerDependent/src/main/java/ghidra/app/extension/datatype/finder/AnonymousVariableAccessDR.java

@@ -0,0 +1,69 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.extension.datatype.finder;
+
+import java.util.List;
+
+import ghidra.app.decompiler.ClangFieldToken;
+import ghidra.app.decompiler.ClangLine;
+import ghidra.app.services.DataTypeReference;
+import ghidra.program.model.data.Composite;
+import ghidra.program.model.data.DataType;
+
+/**
+ * This class represents the use of a field of a {@link Composite} data type <b>where there is
+ * not variable in the Decompiler</b> for that data type.   A normal variable access in the
+ * Decompiler may look like so:
+ * <pre>
+ * 	Foo f;
+ * 	...
+ * 	return f.some_field;
+ * </pre>
+ * 
+ * Alternatively, an anonymous variable access would look like this:
+ * <pre>
+ *	Bar b;
+ * 	...
+ *	return b-><b>foo_array[1].some_field</b>;
+ * </pre>
+ * 
+ * In this case, <code><b>foo_array[1]</b></code> is a <code>Foo</code>, whose 
+ * <code><b>some_field</b></code> is 
+ * being accessed anonymously, since there is no variable of <code>Foo</code> declared 
+ * in the current function. 
+ */
+public class AnonymousVariableAccessDR extends DecompilerReference {
+
+	protected AnonymousVariableAccessDR(ClangLine line, ClangFieldToken token) {
+		super(line, token);
+	}
+
+	@Override
+	public void accumulateMatches(DataType dt, String fieldName, List<DataTypeReference> results) {
+
+		ClangFieldToken field = (ClangFieldToken) sourceToken;
+		DataType fieldDt = field.getDataType();
+		if (!isEqual(dt, fieldDt)) {
+			return;
+		}
+
+		if (field.getText().equals(fieldName)) {
+			results.add(new DataTypeReference(fieldDt, fieldName, getFunction(), getAddress(),
+				getContext()));
+		}
+	}
+
+}

Ghidra/Features/DecompilerDependent/src/main/java/ghidra/app/extension/datatype/finder/DecompilerDataTypeReferenceFinder.java

@@ -20,6 +20,8 @@
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
+import org.apache.commons.collections4.IterableUtils;
+
 import ghidra.app.decompiler.*;
 import ghidra.app.decompiler.component.DecompilerUtils;
 import ghidra.app.decompiler.parallel.*;
@@ -225,13 +227,13 @@ private static class DecompilerDataTypeFinderQCallback
 		private DataType dataType;
 		private String fieldName;
 
-		/** Search for Data Type access only--no field usage */
+		/* Search for Data Type access only--no field usage */
 		DecompilerDataTypeFinderQCallback(Program program, DataType dataType,
 				Consumer<DataTypeReference> callback) {
 			this(program, dataType, null, callback);
 		}
 
-		/** Search for composite field access */
+		/* Search for composite field access */
 		DecompilerDataTypeFinderQCallback(Program program, DataType dataType, String fieldName,
 				Consumer<DataTypeReference> callback) {
 
@@ -336,14 +338,38 @@ private List<DecompilerReference> findVariableReferences(ClangTokenGroup tokens)
 			return result;
 		}
 
+		/**
+		 * Uses the given line to find variables (also parameters and return types) and any 
+		 * accesses to them in that line.   A given variable may be used directly or, as in 
+		 * the case with Composite types, may have one of its fields accessed.  Each result
+		 * found by this method will be at least a variable access and may also itself have
+		 * field accesses.
+		 * 
+		 * <p>Sometimes a line is structured such that there are anonymous variable accesses.  This
+		 * is the case where the field a Composite is being accessed, but the Composite itself is
+		 * not a variable in the current function.  See {@link AnonymousVariableAccessDR} for 
+		 * more details.
+		 * 
+		 * @param line the current line being processed from the Decompiler
+		 * @param results the accumulator into which matches will be placed
+		 */
 		private void findVariablesInLine(ClangLine line, List<DecompilerReference> results) {
 
+			List<ClangToken> allTokens = line.getAllTokens();
+			Iterable<ClangToken> filteredTokens = IterableUtils.filteredIterable(allTokens,
+				token -> {
+					// Only include desirable tokens (this is really just for easier debugging).
+					// Update this filter if the loop below ever needs other types of tokens.
+					return (token instanceof ClangTypeToken) ||
+						(token instanceof ClangVariableToken) || (token instanceof ClangFieldToken);
+				});
+
 			// gather any casts until we can use them (the type they modify will follow)
 			List<DecompilerVariable> castsSoFar = new ArrayList<>();
 
 			VariableDR declaration = null;
 			VariableAccessDR access = null;
-			for (ClangToken token : line.getAllTokens()) {
+			for (ClangToken token : filteredTokens) {
 
 				if (token instanceof ClangTypeToken) {
 
@@ -371,16 +397,15 @@ else if (token instanceof ClangVariableToken) {
 
 					//
 					// Observations: 
-					// 1) 'variableAccess' will be null if we are on a C statement that 
-					//    is a declaration (parameter or variable).  In this case, 'ref' will
-					//    be an instance of VariableDR.
-					// 2) 'variableAccess' will be null the first time a variable is used in
+					// 1) 'access' will be null if we are on a C statement that 
+					//    is a declaration (parameter or variable).  In this case, 
+					//    'declaration' will be an instance of VariableDR.
+					// 2) 'access' will be null the first time a variable is used in
 					//    a statement.
-					// 3) if 'variableAccess' is non-null, but already has a variable assigned, 
+					// 3) if 'access' is non-null, but already has a variable assigned, 
 					//    then this means the current ClangVariableToken represents a new 
 					//    variable access/usage.
 					//
-
 					if (declaration != null) {
 						declaration.setVariable((ClangVariableToken) token);
 						declaration = null;
@@ -415,12 +440,32 @@ else if (token instanceof ClangFieldToken) {
 						}
 					}
 
-					access.addField((ClangFieldToken) token, casts);
+					ClangFieldToken field = (ClangFieldToken) token;
+					if (typesDoNotMatch(access, field)) {
+						// this can happen when a field is used anonymously, such as directly 
+						// after a nested array index operation
+						results.add(new AnonymousVariableAccessDR(line, field));
+						continue;
+					}
+
+					access.addField(field, casts);
 					castsSoFar.clear();
 				}
 			}
 		}
 
+		private boolean typesDoNotMatch(VariableAccessDR access, ClangFieldToken field) {
+
+			DecompilerVariable variable = access.getVariable();
+			if (variable == null) {
+				return false; // should not happen
+			}
+
+			DataType variableDt = variable.getDataType();
+			DataType fieldDt = field.getDataType();
+			return !DecompilerReference.isEqual(variableDt, fieldDt);
+		}
+
 		private VariableAccessDR getLastAccess(List<DecompilerReference> variables) {
 			// for now, assume that the last access will be the last item we added
 			if (variables.isEmpty()) {

Ghidra/Features/DecompilerDependent/src/main/java/ghidra/app/extension/datatype/finder/DecompilerReference.java

@@ -83,6 +83,10 @@ public Address getAddress(DecompilerVariable var) {
 		return var.getAddress();
 	}
 
+	public ClangLine getLine() {
+		return line;
+	}
+
 	protected String getContext() {
 		String context = getContext(variable);
 		return context;

Ghidra/Features/DecompilerDependent/src/main/java/ghidra/app/extension/datatype/finder/DecompilerVariable.java

@@ -52,7 +52,7 @@ public DataType getDataType() {
 		}
 
 		// Note: this is the icky part of the API.  How to know from where to get the data type?
-		HighVariable highVariable = variable.getHighVariable();		
+		HighVariable highVariable = variable.getHighVariable();
 		if (highVariable != null) {
 			return highVariable.getDataType();
 		}
@@ -62,7 +62,7 @@ public DataType getDataType() {
 		if (dataType != null) {
 			return dataType;
 		}
-		
+
 		// Prefer the type of the first input varnode, unless that type is a 'void *'.  
 		// Usually, in that special case, the output varnode has the correct type information. 		
 		PcodeOp op = variable.getPcodeOp();
@@ -104,7 +104,19 @@ private DataType getOutputDataType(PcodeOp op) {
 		}
 
 		Varnode output = op.getOutput();
-		return output.getHigh().getDataType();
+		if (output == null) {
+			// can happen when a variable in volatile memory is used in a write_volatile
+			// pseudo operation
+			return null;
+		}
+
+		HighVariable high = output.getHigh();
+		if (high == null) {
+			// not sure if this can happen; just in case
+			return null;
+		}
+
+		return high.getDataType();
 	}
 
 	private DataType getDataType(Varnode varnode) {