Expose API for running core's string detection on arbitrary buffers

This allows running string detection over an entire shared cache without
having to load all of its files into a binary view.

Author: bdash

binaryninjaapi.h

@@ -5508,6 +5508,63 @@ namespace BinaryNinja {
 		static DerivedString FromAPIObject(BNDerivedString* str, bool owned);
 	};
 
+	/*! Parameters controlling raw string detection, as used by the core strings analysis.
+
+		\see StringDetector
+	*/
+	struct StringDetectionParameters
+	{
+		size_t minStringLength = 4;
+		bool utf8Enabled = true;
+		bool utf16Enabled = true;
+		bool utf32Enabled = true;
+		std::vector<std::string> unicodeBlockNames;
+
+		/*! Builds parameters from the standard string-analysis settings:
+			"analysis.limits.minStringLength" and "analysis.unicode.{blocks,utf8,utf16,utf32}".
+
+			\param settings Settings instance to query, e.g. \c Settings::Instance()
+			\param view Optional view for view-scoped setting values
+			\return Parameters reflecting the given settings
+		*/
+		static StringDetectionParameters FromSettings(Ref<Settings> settings, Ref<BinaryView> view = nullptr);
+	};
+
+	/*! A compiled string detector using the same detection logic as the core strings analysis.
+
+		The detector is immutable once constructed, so a single instance may be shared across threads.
+	*/
+	class StringDetector
+	{
+		BNStringDetector* m_object;
+
+	public:
+		explicit StringDetector(const StringDetectionParameters& params);
+		~StringDetector();
+		StringDetector(const StringDetector&) = delete;
+		StringDetector& operator=(const StringDetector&) = delete;
+		StringDetector(StringDetector&& other) noexcept;
+		StringDetector& operator=(StringDetector&& other) noexcept;
+
+		/*! Detects strings in a raw data buffer.
+
+			Strings must start within the first \c blockLen bytes of \c data but may extend up to
+			\c dataLen bytes, allowing large buffers to be scanned in chunks with a
+			\c BN_MAX_STRING_LENGTH overlap tail. \c lastFoundString (optional, in/out,
+			zero-initialized before the first call) carries overlap state across consecutive chunk
+			calls so strings spanning a chunk boundary are not reported twice.
+
+			\param data Buffer to scan
+			\param dataLen Total number of valid bytes in \c data
+			\param blockLen Number of bytes within which strings may start
+			\param baseAddress Address reported for offset 0 of \c data
+			\param lastFoundString Optional cross-chunk overlap state
+			\return The strings found, with addresses relative to \c baseAddress
+		*/
+		std::vector<BNStringReference> DetectStrings(const void* data, size_t dataLen, size_t blockLen,
+			uint64_t baseAddress, BNStringReference* lastFoundString = nullptr) const;
+	};
+
 	struct QualifiedNameAndType;
 	struct PossibleValueSet;
 	class Metadata;

binaryninjacore.h

@@ -348,6 +348,7 @@ extern "C"
 	typedef struct BNLineFormatter BNLineFormatter;
 	typedef struct BNRenderLayer BNRenderLayer;
 	typedef struct BNStringRef BNStringRef;
+	typedef struct BNStringDetector BNStringDetector;
 	typedef struct BNIndirectBranchInfo BNIndirectBranchInfo;
 	typedef struct BNArchitectureAndAddress BNArchitectureAndAddress;
 	typedef struct BNConstantRenderer BNConstantRenderer;
@@ -5718,6 +5719,32 @@ extern "C"
 	    BNBinaryView* view, uint64_t start, uint64_t len, size_t* count);
 	BINARYNINJACOREAPI void BNFreeStringReferenceList(BNStringReference* strings);
 
+	typedef struct BNStringDetectionParameters
+	{
+		size_t minStringLength;
+		bool utf8Enabled;
+		bool utf16Enabled;
+		bool utf32Enabled;
+		// Unicode block names as accepted by the "analysis.unicode.blocks" setting.
+		const char* const* unicodeBlockNames;
+		size_t unicodeBlockNameCount;
+	} BNStringDetectionParameters;
+
+	// A compiled string detector. Immutable once created, so a single instance may be shared
+	// across threads. Free with BNFreeStringDetector.
+	BINARYNINJACOREAPI BNStringDetector* BNCreateStringDetector(const BNStringDetectionParameters* params);
+	BINARYNINJACOREAPI void BNFreeStringDetector(BNStringDetector* detector);
+
+	// Detects strings starting within the first blockLen bytes of data. Strings may extend up to
+	// dataLen bytes, allowing callers to scan large buffers in chunks with a BN_MAX_STRING_LENGTH
+	// overlap tail. lastFoundString (optional, in/out, zero-initialize before the first call)
+	// carries overlap state across consecutive calls so strings spanning a chunk boundary are not
+	// reported twice. Result addresses are relative to baseAddress. Free the result with
+	// BNFreeStringReferenceList.
+	BINARYNINJACOREAPI BNStringReference* BNStringDetectorDetectStrings(BNStringDetector* detector,
+		const uint8_t* data, size_t dataLen, size_t blockLen, uint64_t baseAddress,
+		BNStringReference* lastFoundString, size_t* count);
+
 	BINARYNINJACOREAPI BNDerivedString* BNGetDerivedStrings(BNBinaryView* view, size_t* count);
 	BINARYNINJACOREAPI BNReferenceSource* BNGetDerivedStringCodeReferences(
 		BNBinaryView* view, BNDerivedString* str, size_t* count, bool limit, size_t maxItems);

binaryview.cpp

@@ -4050,6 +4050,74 @@ vector<BNStringReference> BinaryView::GetStrings(uint64_t start, uint64_t len)
 }
 
 
+StringDetectionParameters StringDetectionParameters::FromSettings(Ref<Settings> settings, Ref<BinaryView> view)
+{
+	StringDetectionParameters params;
+	params.minStringLength = settings->Get<uint64_t>("analysis.limits.minStringLength", view);
+	params.utf8Enabled = settings->Get<bool>("analysis.unicode.utf8", view);
+	params.utf16Enabled = settings->Get<bool>("analysis.unicode.utf16", view);
+	params.utf32Enabled = settings->Get<bool>("analysis.unicode.utf32", view);
+	params.unicodeBlockNames = settings->Get<vector<string>>("analysis.unicode.blocks", view);
+	return params;
+}
+
+
+StringDetector::StringDetector(const StringDetectionParameters& params)
+{
+	BNStringDetectionParameters apiParams;
+	apiParams.minStringLength = params.minStringLength;
+	apiParams.utf8Enabled = params.utf8Enabled;
+	apiParams.utf16Enabled = params.utf16Enabled;
+	apiParams.utf32Enabled = params.utf32Enabled;
+	vector<const char*> blockNames;
+	blockNames.reserve(params.unicodeBlockNames.size());
+	for (const auto& name : params.unicodeBlockNames)
+		blockNames.push_back(name.c_str());
+	apiParams.unicodeBlockNames = blockNames.data();
+	apiParams.unicodeBlockNameCount = blockNames.size();
+	m_object = BNCreateStringDetector(&apiParams);
+}
+
+
+StringDetector::~StringDetector()
+{
+	if (m_object)
+		BNFreeStringDetector(m_object);
+}
+
+
+StringDetector::StringDetector(StringDetector&& other) noexcept : m_object(other.m_object)
+{
+	other.m_object = nullptr;
+}
+
+
+StringDetector& StringDetector::operator=(StringDetector&& other) noexcept
+{
+	if (this != &other)
+	{
+		if (m_object)
+			BNFreeStringDetector(m_object);
+		m_object = other.m_object;
+		other.m_object = nullptr;
+	}
+	return *this;
+}
+
+
+vector<BNStringReference> StringDetector::DetectStrings(const void* data, size_t dataLen, size_t blockLen,
+	uint64_t baseAddress, BNStringReference* lastFoundString) const
+{
+	size_t count = 0;
+	BNStringReference* strings = BNStringDetectorDetectStrings(m_object, static_cast<const uint8_t*>(data),
+		dataLen, blockLen, baseAddress, lastFoundString, &count);
+	vector<BNStringReference> result;
+	result.insert(result.end(), strings, strings + count);
+	BNFreeStringReferenceList(strings);
+	return result;
+}
+
+
 vector<DerivedString> BinaryView::GetDerivedStrings()
 {
 	size_t count;

python/binaryview.py

@@ -32,7 +32,7 @@
 import uuid
 from typing import Callable, Generator, Optional, Union, Tuple, List, Sequence, Mapping, Any, \
 	Iterator, Iterable, KeysView, ItemsView, ValuesView, Dict, overload
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import IntFlag
 
 import collections
@@ -526,6 +526,88 @@ def view(self) -> 'BinaryView':
 		return self._view
 
 
+@dataclass
+class StringDetectionParameters:
+	"""Parameters controlling raw string detection, as used by the core strings analysis."""
+	min_string_length: int = 4
+	utf8_enabled: bool = True
+	utf16_enabled: bool = True
+	utf32_enabled: bool = True
+	unicode_block_names: List[str] = field(default_factory=list)
+
+	@classmethod
+	def from_settings(
+	    cls, settings_obj: Optional['settings.Settings'] = None, view: Optional['BinaryView'] = None
+	) -> 'StringDetectionParameters':
+		"""
+		``from_settings`` builds parameters from the standard string-analysis settings:
+		``analysis.limits.minStringLength`` and ``analysis.unicode.{blocks,utf8,utf16,utf32}``.
+		"""
+		if settings_obj is None:
+			settings_obj = settings.Settings()
+		return cls(
+		    min_string_length=settings_obj.get_integer("analysis.limits.minStringLength", view),
+		    utf8_enabled=settings_obj.get_bool("analysis.unicode.utf8", view),
+		    utf16_enabled=settings_obj.get_bool("analysis.unicode.utf16", view),
+		    utf32_enabled=settings_obj.get_bool("analysis.unicode.utf32", view),
+		    unicode_block_names=settings_obj.get_string_list("analysis.unicode.blocks", view)
+		)
+
+
+@dataclass(frozen=True)
+class DetectedString:
+	"""A string detected by :py:func:`detect_strings_in_block`. ``start`` is relative to the
+	``base_address`` passed to the detector, and ``length`` is in bytes."""
+	type: StringType
+	start: int
+	length: int
+
+
+def detect_strings_in_block(
+    data: bytes, base_address: int = 0, parameters: Optional[StringDetectionParameters] = None
+) -> List[DetectedString]:
+	"""
+	``detect_strings_in_block`` detects strings in a raw data buffer using the same detection logic
+	as the core strings analysis. Unlike :py:meth:`BinaryView.get_strings`, the data does not need to
+	be part of a BinaryView.
+
+	:param data: Buffer to scan
+	:param base_address: Address reported for offset 0 of ``data``
+	:param parameters: Detection parameters; defaults to the current global settings
+	:return: The strings detected
+	"""
+	if parameters is None:
+		parameters = StringDetectionParameters.from_settings()
+
+	params = core.BNStringDetectionParameters()
+	params.minStringLength = parameters.min_string_length
+	params.utf8Enabled = parameters.utf8_enabled
+	params.utf16Enabled = parameters.utf16_enabled
+	params.utf32Enabled = parameters.utf32_enabled
+	block_names = (ctypes.c_char_p * len(parameters.unicode_block_names))()
+	for i, name in enumerate(parameters.unicode_block_names):
+		block_names[i] = core.cstr(name)
+	params.unicodeBlockNames = ctypes.cast(block_names, ctypes.POINTER(ctypes.c_char_p))
+	params.unicodeBlockNameCount = len(parameters.unicode_block_names)
+
+	detector = core.BNCreateStringDetector(params)
+	assert detector is not None, "core.BNCreateStringDetector returned None"
+	result = []
+	try:
+		buf = (ctypes.c_ubyte * len(data)).from_buffer_copy(data)
+		count = ctypes.c_ulonglong()
+		strings = core.BNStringDetectorDetectStrings(detector, buf, len(data), len(data), base_address, None, count)
+		assert strings is not None, "core.BNStringDetectorDetectStrings returned None"
+		try:
+			for i in range(count.value):
+				result.append(DetectedString(StringType(strings[i].type), strings[i].start, strings[i].length))
+		finally:
+			core.BNFreeStringReferenceList(strings)
+	finally:
+		core.BNFreeStringDetector(detector)
+	return result
+
+
 class StringRef:
 	"""Deduplicated reference to a string owned by the Binary Ninja core. Use `str` or `bytes` to convert
 	this to a standard Python string or sequence of bytes."""

rust/src/lib.rs

@@ -79,6 +79,7 @@ pub mod section;
 pub mod segment;
 pub mod settings;
 pub mod string;
+pub mod string_detection;
 pub mod symbol;
 pub mod tags;
 pub mod template_simplifier;