flatlaf-natives-windows: fixed memory allocation error handling (issue #591)

flatlaf-natives/flatlaf-natives-windows/src/main/cpp/FlatWndProc.cpp

@@ -99,8 +99,11 @@ HWND FlatWndProc::install( JNIEnv *env, jobject obj, jobject window ) {
 		return 0;
 
 	// create HWND map
-	if( hwndMap == NULL )
+	if( hwndMap == NULL ) {
 		hwndMap = new HWNDMap();
+		if( hwndMap == NULL )
+			return 0;
+	}
 
 	// get window handle
 	HWND hwnd = getWindowHandle( env, window );
@@ -108,10 +111,17 @@ HWND FlatWndProc::install( JNIEnv *env, jobject obj, jobject window ) {
 		return 0;
 
 	FlatWndProc* fwp = new FlatWndProc();
+	if( fwp == NULL )
+		return 0;
+
+	if( !hwndMap->put( hwnd, fwp ) ) {
+		delete fwp;
+		return 0;
+	}
+
 	env->GetJavaVM( &fwp->jvm );
 	fwp->obj = env->NewGlobalRef( obj );
 	fwp->hwnd = hwnd;
-	hwndMap->put( hwnd, fwp );
 
 	// replace window procedure
 	fwp->defaultWndProc = reinterpret_cast<WNDPROC>(

flatlaf-natives/flatlaf-natives-windows/src/main/cpp/HWNDMap.cpp

@@ -43,8 +43,8 @@ public:
 
 HWNDMap::HWNDMap() {
 	size = 0;
-	capacity = DEFAULT_CAPACITY;
-	table = new Entry[capacity];
+	capacity = 0;
+	table = NULL;
 
 	::InitializeCriticalSection( &criticalSection );
 
@@ -58,7 +58,7 @@ LPVOID HWNDMap::get( HWND key ) {
 	return (index >= 0) ? table[index].value : NULL;
 }
 
-void HWNDMap::put( HWND key, LPVOID value ) {
+bool HWNDMap::put( HWND key, LPVOID value ) {
 	LOCK lock( &criticalSection );
 
 	int index = binarySearch( key );
@@ -68,9 +68,10 @@ void HWNDMap::put( HWND key, LPVOID value ) {
 		table[index].value = value;
 	} else {
 		// insert new key
-		ensureCapacity( size + 1 );
+		if( !ensureCapacity() )
+			return false;
 
-		// make roor for new entry
+		// make room for new entry
 		index = -(index + 1);
 		for( int i = size - 1; i >= index; i-- )
 			table[i + 1] = table[i];
@@ -82,6 +83,7 @@ void HWNDMap::put( HWND key, LPVOID value ) {
 	}
 
 //	dump( "put" );
+	return true;
 }
 
 void HWNDMap::remove( HWND key ) {
@@ -102,6 +104,9 @@ void HWNDMap::remove( HWND key ) {
 }
 
 int HWNDMap::binarySearch( HWND key ) {
+	if( table == NULL )
+		return -1;
+
 	__int64 ikey = reinterpret_cast<__int64>( key );
 	int low = 0;
 	int high = size - 1;
@@ -121,23 +126,37 @@ int HWNDMap::binarySearch( HWND key ) {
 	return -(low + 1);
 }
 
-void HWNDMap::ensureCapacity( int minCapacity ) {
+bool HWNDMap::ensureCapacity() {
+	if( table == NULL ) {
+		table = new Entry[DEFAULT_CAPACITY];
+		if( table == NULL )
+			return false;
+
+		capacity = DEFAULT_CAPACITY;
+		return true;
+	}
+
+	// check capacity
+	int minCapacity = size + 1;
 	if( minCapacity <= capacity )
-		return;
+		return true;
 
 	// allocate new table
 	int newCapacity = minCapacity + INCREASE_CAPACITY;
 	Entry* newTable = new Entry[newCapacity];
+	if( newTable == NULL )
+		return false;
 
 	// copy old table to new table
 	for( int i = 0; i < capacity; i++ )
 		newTable[i] = table[i];
 
 	// delete old table
-	delete table;
+	delete[] table;
 
 	table = newTable;
 	capacity = newCapacity;
+	return true;
 }
 
 /*

flatlaf-natives/flatlaf-natives-windows/src/main/cpp/HWNDMap.h

@@ -42,12 +42,12 @@ public:
 	HWNDMap();
 
 	LPVOID get( HWND key );
-	void put( HWND key, LPVOID value );
+	bool put( HWND key, LPVOID value );
 	void remove( HWND key );
 
 private:
 	int binarySearch( HWND key );
-	void ensureCapacity( int newCapacity );
+	bool ensureCapacity();
 
 //	void dump( char* msg );
 };

flatlaf-natives/flatlaf-natives-windows/src/main/cpp/Runtime.cpp

@@ -42,14 +42,23 @@ BOOL WINAPI _DllMainCRTStartup( HINSTANCE instance, DWORD reason, LPVOID reserve
 }
 
 void* __cdecl operator new( size_t cb ) {
+//	printf( "new %d\n", cb );
 	return ::HeapAlloc( ::GetProcessHeap(), HEAP_ZERO_MEMORY, cb );
 }
 
 void* __cdecl operator new[]( size_t cb ) {
+//	printf( "new[] %d\n", cb );
 	return ::HeapAlloc( ::GetProcessHeap(), HEAP_ZERO_MEMORY, cb );
 }
 
 void __cdecl operator delete( void* pv, size_t cb ) {
+//	printf( "delete %p %d\n", pv, cb );
+	if( pv != NULL )
+		::HeapFree( ::GetProcessHeap(), 0, pv );
+}
+
+void __cdecl operator delete[]( void* pv ) {
+//	printf( "delete[] %p\n", pv );
 	if( pv != NULL )
 		::HeapFree( ::GetProcessHeap(), 0, pv );
 }