From 6f3acbac0d4de2ea45f6f980500f42eaad3bcf00 Mon Sep 17 00:00:00 2001
From: Yann Collet <yann.collet.73@gmail.com>
Date: Fri, 12 Feb 2016 20:19:48 +0100
Subject: [PATCH] notificationLevel into ZDICT_param_t
---
lib/zdict.c | 4 ++--
lib/zdict_static.h | 28 ++--------------------------
programs/dibio.c | 14 +++++++++++++-
programs/dibio.h | 11 -----------
programs/fileio.c | 18 +++++++++---------
programs/zstdcli.c | 4 ++--
6 files changed, 28 insertions(+), 51 deletions(-)
diff --git a/lib/zdict.c b/lib/zdict.c
index ec2ffbb70..cd58ab8a9 100644
--- a/lib/zdict.c
+++ b/lib/zdict.c
@@ -102,7 +102,6 @@ static const size_t g_min_fast_dictContent = 192;
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
-void ZDICT_setNotificationLevel(unsigned l) { g_displayLevel=l; }
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if (ZDICT_GetMilliSpan(g_time) > refreshRate) \
@@ -111,7 +110,7 @@ void ZDICT_setNotificationLevel(unsigned l) { g_displayLevel=l; }
static const unsigned refreshRate = 300;
static clock_t g_time = 0;
-void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
+static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
{
const BYTE* const b = (const BYTE*)ptr;
size_t u;
@@ -813,6 +812,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; }
if (!dictList) return ERROR(memory_allocation);
ZDICT_initDictItem(dictList);
+ g_displayLevel = params.notificationLevel;
if (selectivity==0) selectivity = g_selectivity_default;
if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
diff --git a/lib/zdict_static.h b/lib/zdict_static.h
index e1ef73136..e5f909ac7 100644
--- a/lib/zdict_static.h
+++ b/lib/zdict_static.h
@@ -53,6 +53,8 @@ extern "C" {
typedef struct {
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
+ unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+ unsigned reserved[3]; /* space for future parameters */
} ZDICT_params_t;
@@ -71,32 +73,6 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit
ZDICT_params_t parameters);
-/*-*************************************
-* Helper functions
-***************************************/
-/*! ZDICT_setNotificationLevel() :
- Set amount of notification to be displayed on the console.
- default : 0 = no console notification.
- 1 = errors; 2 = notifications; 3 = details; 4 = debug;
- Note : not thread-safe (uses a global constant)
-*/
-void ZDICT_setNotificationLevel(unsigned l);
-
-
-/*-*************************************
-* Private functions
-***************************************/
-/*! ZDICT_trainFromBuffer_unsafe() :
- Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
- note : `samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
- @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
- or an error code.
-*/
-size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
- ZDICT_params_t parameters);
-
-
#if defined (__cplusplus)
}
#endif
diff --git a/programs/dibio.c b/programs/dibio.c
index 7b6611536..646fe2c60 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -85,7 +85,6 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
-void DiB_setNotificationLevel(unsigned l) { g_displayLevel=l; ZDICT_setNotificationLevel(l); }
/*-*************************************
@@ -216,6 +215,18 @@ static void DiB_saveDict(const char* dictFileName,
}
+/*! ZDICT_trainFromBuffer_unsafe() :
+ Strictly Internal use only !!
+ Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
+ `samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+ or an error code.
+*/
+size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t parameters);
+
+
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
const char** fileNamesTable, unsigned nbFiles,
ZDICT_params_t params)
@@ -229,6 +240,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
int result = 0;
/* init */
+ g_displayLevel = params.notificationLevel;
benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
diff --git a/programs/dibio.h b/programs/dibio.h
index 4d24cc16a..0ccec4135 100644
--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -49,15 +49,4 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
ZDICT_params_t parameters);
-/*-*************************************
-* Helper functions
-***************************************/
-/*! DiB_setNotificationLevel
- Set amount of notification to be displayed on the console.
- default initial value : 0 = no console notification.
- Note : not thread-safe (use a global constant)
-*/
-void DiB_setNotificationLevel(unsigned l);
-
-
#endif
diff --git a/programs/fileio.c b/programs/fileio.c
index 898f47202..003e1bd51 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -52,7 +52,7 @@
#define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */
-/* *************************************
+/*-*************************************
* Includes
***************************************/
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
@@ -68,12 +68,12 @@
#include "zbuff_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-# include "zstd_legacy.h" /* legacy */
-# include "fileio_legacy.h" /* legacy */
+# include "zstd_legacy.h" /* ZSTD_isLegacy */
+# include "fileio_legacy.h" /* FIO_decompressLegacyFrame */
#endif
-/* *************************************
+/*-*************************************
* OS-specific Includes
***************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
@@ -92,7 +92,7 @@
#endif
-/* *************************************
+/*-*************************************
* Constants
***************************************/
#define KB *(1U<<10)
@@ -112,12 +112,13 @@
#define BLOCKSIZE (128 KB)
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
-#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
-#define FSE_CHECKSUM_SEED 0
+#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
+#define FSE_CHECKSUM_SEED 0
#define CACHELINE 64
-#define MAX_DICT_SIZE (512 KB)
+#define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */
+
/* *************************************
* Macros
@@ -138,7 +139,6 @@ static clock_t g_time = 0;
* Local Parameters
***************************************/
static U32 g_overwrite = 0;
-
void FIO_overwriteMode(void) { g_overwrite=1; }
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 22c8c8182..c8a372828 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -133,7 +133,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -c : force write to standard output, even if it is the console\n");
#ifndef ZSTD_NODICT
DISPLAY( "Dictionary builder :\n");
- DISPLAY( "--train : Create a dictionary from a set of files \n");
+ DISPLAY( "--train : create a dictionary from a training set of files \n");
DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName);
DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
@@ -366,7 +366,7 @@ int main(int argCount, const char** argv)
ZDICT_params_t dictParams;
dictParams.compressionLevel = dictCLevel;
dictParams.selectivityLevel = dictSelect;
- DiB_setNotificationLevel(displayLevel);
+ dictParams.notificationLevel = displayLevel;
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
#endif
goto _end;
--
GitLab