gcc/lto-compress.cc

/* LTO IL compression streams.

   Copyright (C) 2009-2024 Free Software Foundation, Inc.
   Contributed by Simon Baldwin <simonb@google.com>

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3, or (at your option)
any later version.

GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
License for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "tree.h"
#include "gimple.h"
#include "cgraph.h"
#include "lto-streamer.h"
/* zlib.h includes other system headers.  Those headers may test feature
   test macros.  config.h may define feature test macros.  For this reason,
   zlib.h needs to be included after, rather than before, config.h and
   system.h.  */
#include <zlib.h>
#include "lto-compress.h"
#include "timevar.h"

#ifdef HAVE_ZSTD_H
#include <zstd.h>
#endif

/* Compression stream structure, holds the flush callback and opaque token,
   the buffered data, and a note of whether compressing or uncompressing.  */

struct lto_compression_stream
{
  void (*callback) (const char *, unsigned, void *);
  void *opaque;
  char *buffer;
  size_t bytes;
  size_t allocation;
  bool is_compression;
};

/* Overall compression constants for zlib.  */

static const size_t Z_BUFFER_LENGTH = 4096;
static const size_t MIN_STREAM_ALLOCATION = 1024;

/* For zlib, allocate SIZE count of ITEMS and return the address, OPAQUE
   is unused.  */

static void *
lto_zalloc (void *opaque, unsigned items, unsigned size)
{
  gcc_assert (opaque == Z_NULL);
  return xmalloc (items * size);
}

/* For zlib, free memory at ADDRESS, OPAQUE is unused.  */

static void
lto_zfree (void *opaque, void *address)
{
  gcc_assert (opaque == Z_NULL);
  free (address);
}

/* Return a zlib compression level that zlib will not reject.  Normalizes
   the compression level from the command line flag, clamping non-default
   values to the appropriate end of their valid range.  */

static int
lto_normalized_zlib_level (void)
{
  int level = flag_lto_compression_level;

  if (level != Z_DEFAULT_COMPRESSION)
    {
      if (level < Z_NO_COMPRESSION)
	level = Z_NO_COMPRESSION;
      else if (level > Z_BEST_COMPRESSION)
	level = Z_BEST_COMPRESSION;
    }

  return level;
}

/* Free the buffer and memory associated with STREAM.  */

static void
lto_destroy_compression_stream (struct lto_compression_stream *stream)
{
  free (stream->buffer);
  free (stream);
}

#ifdef HAVE_ZSTD_H
/* Return a zstd compression level that zstd will not reject.  Normalizes
   the compression level from the command line flag, clamping non-default
   values to the appropriate end of their valid range.  */

static int
lto_normalized_zstd_level (void)
{
  int level = flag_lto_compression_level;

  if (level < 0)
    level = 0;
  else if (level > ZSTD_maxCLevel ())
    level = ZSTD_maxCLevel ();

  return level;
}

/* Compress STREAM using ZSTD algorithm.  */

static void
lto_compression_zstd (struct lto_compression_stream *stream)
{
  unsigned char *cursor = (unsigned char *) stream->buffer;
  size_t size = stream->bytes;

  timevar_push (TV_IPA_LTO_COMPRESS);
  size_t const outbuf_length = ZSTD_compressBound (size);
  char *outbuf = (char *) xmalloc (outbuf_length);

  size_t const csize = ZSTD_compress (outbuf, outbuf_length, cursor, size,
				      lto_normalized_zstd_level ());

  if (ZSTD_isError (csize))
    internal_error ("compressed stream: %s", ZSTD_getErrorName (csize));

  lto_stats.num_compressed_il_bytes += csize;
  stream->callback (outbuf, csize, NULL);

  lto_destroy_compression_stream (stream);
  free (outbuf);
  timevar_pop (TV_IPA_LTO_COMPRESS);
}

/* Uncompress STREAM using ZSTD algorithm.  */

static void
lto_uncompression_zstd (struct lto_compression_stream *stream)
{
  unsigned char *cursor = (unsigned char *) stream->buffer;
  size_t size = stream->bytes;

  timevar_push (TV_IPA_LTO_DECOMPRESS);
  unsigned long long const rsize = ZSTD_getFrameContentSize (cursor, size);
  if (rsize == ZSTD_CONTENTSIZE_ERROR)
    internal_error ("original not compressed with zstd");
  else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN)
    internal_error ("original size unknown");

  char *outbuf = (char *) xmalloc (rsize);
  size_t const dsize = ZSTD_decompress (outbuf, rsize, cursor, size);

  if (ZSTD_isError (dsize))
    internal_error ("decompressed stream: %s", ZSTD_getErrorName (dsize));

  lto_stats.num_uncompressed_il_bytes += dsize;
  stream->callback (outbuf, dsize, stream->opaque);

  lto_destroy_compression_stream (stream);
  free (outbuf);
  timevar_pop (TV_IPA_LTO_DECOMPRESS);
}

#endif

/* Create a new compression stream, with CALLBACK flush function passed
   OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing.  */

static struct lto_compression_stream *
lto_new_compression_stream (void (*callback) (const char *, unsigned, void *),
			    void *opaque, bool is_compression)
{
  struct lto_compression_stream *stream
    = (struct lto_compression_stream *) xmalloc (sizeof (*stream));

  memset (stream, 0, sizeof (*stream));
  stream->callback = callback;
  stream->opaque = opaque;
  stream->is_compression = is_compression;

  return stream;
}

/* Append NUM_CHARS from address BASE to STREAM.  */

static void
lto_append_to_compression_stream (struct lto_compression_stream *stream,
				  const char *base, size_t num_chars)
{
  size_t required = stream->bytes + num_chars;

  if (stream->allocation < required)
    {
      if (stream->allocation == 0)
	stream->allocation = MIN_STREAM_ALLOCATION;
      while (stream->allocation < required)
	stream->allocation *= 2;

      stream->buffer = (char *) xrealloc (stream->buffer, stream->allocation);
    }

  memcpy (stream->buffer + stream->bytes, base, num_chars);
  stream->bytes += num_chars;
}

/* Return a new compression stream, with CALLBACK flush function passed
   OPAQUE token.  */

struct lto_compression_stream *
lto_start_compression (void (*callback) (const char *, unsigned, void *),
		       void *opaque)
{
  return lto_new_compression_stream (callback, opaque, true);
}

/* Append NUM_CHARS from address BASE to STREAM.  */

void
lto_compress_block (struct lto_compression_stream *stream,
		    const char *base, size_t num_chars)
{
  gcc_assert (stream->is_compression);

  lto_append_to_compression_stream (stream, base, num_chars);
  lto_stats.num_output_il_bytes += num_chars;
}

static void ATTRIBUTE_UNUSED
lto_compression_zlib (struct lto_compression_stream *stream)
{
  unsigned char *cursor = (unsigned char *) stream->buffer;
  size_t remaining = stream->bytes;
  const size_t outbuf_length = Z_BUFFER_LENGTH;
  unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length);
  z_stream out_stream;
  int status;

  gcc_assert (stream->is_compression);

  timevar_push (TV_IPA_LTO_COMPRESS);

  out_stream.next_out = outbuf;
  out_stream.avail_out = outbuf_length;
  out_stream.next_in = cursor;
  out_stream.avail_in = remaining;
  out_stream.zalloc = lto_zalloc;
  out_stream.zfree = lto_zfree;
  out_stream.opaque = Z_NULL;

  status = deflateInit (&out_stream, lto_normalized_zlib_level ());
  if (status != Z_OK)
    internal_error ("compressed stream: %s", zError (status));

  do
    {
      size_t in_bytes, out_bytes;

      status = deflate (&out_stream, Z_FINISH);
      if (status != Z_OK && status != Z_STREAM_END)
	internal_error ("compressed stream: %s", zError (status));

      in_bytes = remaining - out_stream.avail_in;
      out_bytes = outbuf_length - out_stream.avail_out;

      stream->callback ((const char *) outbuf, out_bytes, stream->opaque);
      lto_stats.num_compressed_il_bytes += out_bytes;

      cursor += in_bytes;
      remaining -= in_bytes;

      out_stream.next_out = outbuf;
      out_stream.avail_out = outbuf_length;
      out_stream.next_in = cursor;
      out_stream.avail_in = remaining;
    }
  while (status != Z_STREAM_END);

  status = deflateEnd (&out_stream);
  if (status != Z_OK)
    internal_error ("compressed stream: %s", zError (status));

  lto_destroy_compression_stream (stream);
  free (outbuf);
  timevar_pop (TV_IPA_LTO_COMPRESS);
}

void
lto_end_compression (struct lto_compression_stream *stream)
{
#ifdef HAVE_ZSTD_H
  lto_compression_zstd (stream);
#else
  lto_compression_zlib (stream);
#endif
}

/* Return a new uncompression stream, with CALLBACK flush function passed
   OPAQUE token.  */

struct lto_compression_stream *
lto_start_uncompression (void (*callback) (const char *, unsigned, void *),
			 void *opaque)
{
  return lto_new_compression_stream (callback, opaque, false);
}

/* Append NUM_CHARS from address BASE to STREAM.  */

void
lto_uncompress_block (struct lto_compression_stream *stream,
		      const char *base, size_t num_chars)
{
  gcc_assert (!stream->is_compression);

  lto_append_to_compression_stream (stream, base, num_chars);
  lto_stats.num_input_il_bytes += num_chars;
}

static void
lto_uncompression_zlib (struct lto_compression_stream *stream)
{
  unsigned char *cursor = (unsigned char *) stream->buffer;
  size_t remaining = stream->bytes;
  const size_t outbuf_length = Z_BUFFER_LENGTH;
  unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length);

  gcc_assert (!stream->is_compression);
  timevar_push (TV_IPA_LTO_DECOMPRESS);

  while (remaining > 0)
    {
      z_stream in_stream;
      size_t out_bytes;
      int status;

      in_stream.next_out = outbuf;
      in_stream.avail_out = outbuf_length;
      in_stream.next_in = cursor;
      in_stream.avail_in = remaining;
      in_stream.zalloc = lto_zalloc;
      in_stream.zfree = lto_zfree;
      in_stream.opaque = Z_NULL;

      status = inflateInit (&in_stream);
      if (status != Z_OK)
	internal_error ("compressed stream: %s", zError (status));

      do
	{
	  size_t in_bytes;

	  status = inflate (&in_stream, Z_SYNC_FLUSH);
	  if (status != Z_OK && status != Z_STREAM_END)
	    internal_error ("compressed stream: %s", zError (status));

	  in_bytes = remaining - in_stream.avail_in;
	  out_bytes = outbuf_length - in_stream.avail_out;

	  stream->callback ((const char *) outbuf, out_bytes, stream->opaque);
	  lto_stats.num_uncompressed_il_bytes += out_bytes;

	  cursor += in_bytes;
	  remaining -= in_bytes;

	  in_stream.next_out = outbuf;
	  in_stream.avail_out = outbuf_length;
	  in_stream.next_in = cursor;
	  in_stream.avail_in = remaining;
	}
      while (!(status == Z_STREAM_END && out_bytes == 0));

      status = inflateEnd (&in_stream);
      if (status != Z_OK)
	internal_error ("compressed stream: %s", zError (status));
    }

  lto_destroy_compression_stream (stream);
  free (outbuf);
  timevar_pop (TV_IPA_LTO_DECOMPRESS);
}

void
lto_end_uncompression (struct lto_compression_stream *stream,
		       lto_compression compression)
{
#ifdef HAVE_ZSTD_H
  if (compression == ZSTD)
    {
      lto_uncompression_zstd (stream);
      return;
    }
#endif
  if (compression == ZSTD)
    fatal_error (UNKNOWN_LOCATION, "compiler does not support ZSTD LTO compression");

  lto_uncompression_zlib (stream);
}