Homebrew Compressing buffer with Zlib

[BullyWiiPlaza]

I want to compress a data buffer before sending it over the network for performance reasons. For this I decided to implement Zlib. However, I'm getting a MEM_ERROR return code (-4). I closely followed the implementation from here since it seemed reasonably easy and it matches up nicely with what the Wii U has (compress and compress2 functions).

C server code:

case 0x09: { /* read_memory_compressed */
    int startingAddress = 0x10000000; /* Hardcoded example */
    int length = 0x1000;

    // Setup raw data buffer
    void *rawBuffer = malloc(length);
    ASSERT_ALLOCATED(rawBuffer, "Raw buffer")
    memcpy(rawBuffer, (const void *) startingAddress, length);

    // Setup compressed buffer
    int compressedBufferSize = (int) (length + (length * 0.1) + 12) + 1;
    void *compressedBuffer = malloc(compressedBufferSize);
    ASSERT_ALLOCATED(compressedBuffer, "Compressed buffer")

    int destinationBufferSize;
    int status = compress2((char *) compressedBuffer, &destinationBufferSize, (const char *) rawBuffer, length, Z_BEST_COMPRESSION);

    ret = sendwait(bss, clientfd, &status, 4);
    ASSERT_FUNCTION_SUCCEEDED(ret, "sendwait (status)")

    if (status == Z_OK) {
        // Send the compressed buffer size and content
        ((int *) buffer)[0] = destinationBufferSize;
        memcpy(buffer + 4, compressedBuffer, destinationBufferSize);

        ret = sendwait(bss, clientfd, buffer, 4 + destinationBufferSize);
        ASSERT_FUNCTION_SUCCEEDED(ret, "sendwait (Compressed data)")
    }

    free(rawBuffer);
    free(compressedBuffer);

    break;
}

Java client code:

public byte[] readCompressedBytes() throws IOException
{
    try (CloseableReentrantLock ignored = reentrantLock.acquire())
    {
        sendCommand(Command.READ_MEMORY_COMPRESSED);
        dataSender.flush();

        ZLibReturnCode status = ZLibReturnCode.parse(dataReceiver.readInt());

        if (status.equals(ZLibReturnCode.OK))
        {
            int compressedSize = dataReceiver.readInt();

            byte[] bytes = new byte[compressedSize];
            dataReceiver.readFully(bytes);

            return bytes;
        }

        throw new IllegalStateException("Compression returned with status: " + status);
    }
}

Exception in thread "main" java.lang.IllegalStateException: Compression returned with status: MEM_ERROR

Note that the Wii U implements it in zlib125.rpl.

Any help is greatly appreciated, thank you.

@QuarkTheAwesome
@dimok
@Maschell
@wj44
@NWPlayer123

 

[Maschell]

Where is that "buffer" coming from in the C function?

Looks like the compressionBuffer is too small. Did you try to increase it?

 

[BullyWiiPlaza]

With double the size of the output buffer the same error is still being returned.

Instead of the compress() function I now tried using the inflate() function. For this I have to include zlib.h in for example pygecko.c to get access to all definitions and functions. If I do that, it however gives me the following compilation error:

In file included from d:/Consoles/WiiU/Applications/Browser/tcpgecko/src/pygecko.c:5:0:
d:/Consoles/WiiU/Applications/Browser/tcpgecko/src/dynamic_libs/os_functions.h:65:41: error: expected declaration specifiers or '...' before '(' token
#define SECS_TO_TICKS(sec)              (((unsigned long long)(sec)) * (BUS_SPEED/4))
                                         ^
d:/Consoles/WiiU/Applications/Browser/tcpgecko/src/dynamic_libs/os_functions.h:67:41: error: expected declaration specifiers or '...' before '(' token
#define MICROSECS_TO_TICKS(usec)        (SECS_TO_TICKS(usec) / 1000000)
                                         ^

I'm not sure how to fix them since these macros are being used and cannot just be deleted. What is wrong? Without the import it works fine and I don't see a name clash. Thank you

Here is the inflate Zlib code I came up with though but it can't be compiled due to the error above. I might be fairly correct already:

case 0x09: { /* read_memory_compressed */
    // Receive the starting address and length
    ret = recvwait(bss, clientfd, buffer, 4 + 4);
    CHECK_ERROR(ret < 0)
    int startingAddress = ((int *) buffer)[0];
    unsigned int length = ((unsigned int *) buffer)[1];

    // Setup raw data buffer
    void *inputBuffer = malloc(length);
    ASSERT_ALLOCATED(inputBuffer, "Raw buffer")
    memcpy(inputBuffer, (const void *) startingAddress, length);

    // Setup output buffer using system memory due to out of memory issues
    unsigned int outputBufferSize = length * 2;
    void *outputBuffer = (void *) OSAllocFromSystem(outputBufferSize, 0x4);

    z_stream stream;
    memset(&stream, 0, sizeof(stream));

    stream.zalloc = Z_NULL;
    stream.zfree = Z_NULL;
    stream.opaque = Z_NULL;

    // Initialize the stream struct
    int ret = deflateInit(&stream, Z_BEST_COMPRESSION);
    ASSERT_INTEGER(ret, Z_OK, "deflateInit");

    // Supply the data
    stream.avail_in = length;
    stream.next_in = (Bytef *) inputBuffer;
    stream.avail_out = outputBufferSize;
    stream.next_out = (Bytef *) outputBuffer;

    // Deflate and finish
    ret = deflate(&stream, Z_FINISH);
    ASSERT_INTEGER(ret, Z_OK, "deflate");
    ret = deflateEnd(&stream);
    ASSERT_INTEGER(ret, Z_OK, "deflateEnd");

    int deflatedSize = (int) ((void *) stream.next_out - outputBuffer);

    // Send the compressed buffer size and content
    ((int *) buffer)[0] = deflatedSize;
    memcpy(buffer + 4, outputBuffer, deflatedSize);
    ret = sendwait(bss, clientfd, buffer, 4 + deflatedSize);
    ASSERT_FUNCTION_SUCCEEDED(ret, "sendwait (Compressed data)")

    // Clean up again
    free(inputBuffer);
    OSFreeToSystem(outputBuffer);

    break;
}