From d3f2254b58a743422a00544ebc6327aa03cb88a4 Mon Sep 17 00:00:00 2001 From: Nathan Moinvaziri Date: Tue, 5 May 2020 14:56:47 -0700 Subject: [PATCH] Compile each zlib variant with pigz independently. Copy necessary files from pigz dir and then build. Use mainline pigz and zlib repositories. Added rebuild and toolchain args to compile script. --- _a_compile_dynamic.py | 60 --- a_compile.py | 115 ++-- pigz/CMakeLists.txt | 67 +++ pigz/win/dirent.h | 1160 +++++++++++++++++++++++++++++++++++++++++ pigz/win/sys/time.h | 32 ++ pigz/win/unistd.h | 77 +++ 6 files changed, 1420 insertions(+), 91 deletions(-) delete mode 100755 _a_compile_dynamic.py create mode 100644 pigz/CMakeLists.txt create mode 100644 pigz/win/dirent.h create mode 100644 pigz/win/sys/time.h create mode 100644 pigz/win/unistd.h diff --git a/_a_compile_dynamic.py b/_a_compile_dynamic.py deleted file mode 100755 index 0dcdd4f..0000000 --- a/_a_compile_dynamic.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -import os -import stat -import shutil -import subprocess -import platform -import _install_corpus as corpus - - -def rmtree(top): - """Delete folder and contents: shutil.rmtree has issues with read-only files on Windows""" - - for (root, dirs, files) in os.walk(top, topdown=False): - for name in files: - filename = os.path.join(root, name) - os.chmod(filename, stat.S_IWUSR) - os.remove(filename) - for name in dirs: - os.rmdir(os.path.join(root, name)) - os.rmdir(top) - - -def compile_pigz(): - """compile system variant of pigz""" - - method = 'System' - basedir = os.getcwd() - exedir = os.path.join(basedir, 'exe') - if os.path.isdir(exedir): - rmtree(exedir) - try: - os.mkdir(exedir) - except OSError: - print('Unable to create folder "' + exedir +'"') - pigzdir = './pigz' - if os.path.isdir(pigzdir): - rmtree(pigzdir) - - # cmd='git clone https://github.com/neurolabusc/pigz' - - cmd = 'git clone https://github.com/madler/pigz' - subprocess.call(cmd, shell=True) - pigzdir = os.path.join(basedir, 'pigz') - pigzexe = os.path.join(pigzdir, 'pigz') - ext = '' - if platform.system() == 'Windows': - ext = '.exe' - pigzexe = pigzexe + ext - os.chdir(pigzdir) - cmd = 'make' - subprocess.call(cmd, shell=True) - outnm = os.path.join(exedir, 'pigz' + method + ext) - print(pigzexe + '->' + outnm) - shutil.move(pigzexe, outnm) - - -if __name__ == '__main__': - corpus.install_neuro_corpus() - compile_pigz() diff --git a/a_compile.py b/a_compile.py index acff4ff..9c465cb 100755 --- a/a_compile.py +++ b/a_compile.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os +import argparse import stat import shutil import subprocess @@ -8,6 +9,11 @@ import zipfile from distutils.dir_util import copy_tree +parser = argparse.ArgumentParser(description='Pigz script') +parser.add_argument('--rebuild', help='Rebuild', action='store_const', const=True, default=None) +args, unknown = parser.parse_known_args() + + def rmtree(top): """Delete folder and contents: shutil.rmtree has issues with read-only files on Windows""" @@ -20,6 +26,7 @@ def rmtree(top): os.rmdir(os.path.join(root, name)) os.rmdir(top) + def install_silesia_corpus(): """Install popular Silesia corpus""" @@ -30,9 +37,9 @@ def install_silesia_corpus(): try: os.mkdir(corpusdir) except OSError: - print('Creation of the directory {} failed' .format(corpusdir) ) + print('Creation of the directory {} failed' .format(corpusdir)) cmd = 'git clone https://github.com/MiloszKrajewski/SilesiaCorpus silesia' - print("Installing "+corpusdir); + print("Installing "+corpusdir) subprocess.call(cmd, shell=True) os.chdir(corpusdir) fnm = 'README.md' @@ -50,6 +57,7 @@ def install_silesia_corpus(): os.remove(file_name) # delete zipped file os.chdir(basedir) + def install_neuro_corpus(): """Install neuroimaging corpus""" @@ -60,7 +68,7 @@ def install_neuro_corpus(): try: os.mkdir(corpusdir) except OSError: - print('Creation of the directory {} failed' .format(exedir) ) + print('Creation of the directory {} failed' .format(exedir)) cmd = 'git clone https://github.com/neurolabusc/zlib-bench.git' subprocess.call(cmd, shell=True) indir = os.path.join(basedir, 'zlib-bench', 'corpus') @@ -71,51 +79,96 @@ def install_neuro_corpus(): indir = os.path.join(basedir, 'zlib-bench') rmtree(indir) -def compile_pigz(): + +def compile_pigz(rebuild=True): """compile variants of pigz""" - methods = ['ng'] - if platform.system() == 'Windows': - methods = ['Cloudflare', 'ng'] + methods = [ + {'name': 'madler', + 'repository': 'https://github.com/madler/zlib', + 'branch': None}, + {'name': 'cloudflare', + 'repository': 'https://github.com/cloudflare/zlib', # Only supports 64-bit builds + 'branch': None}, + {'name': 'ng', + 'repository': 'https://github.com/zlib-ng/zlib-ng', + 'branch': 'develop'} + ] basedir = os.getcwd() exedir = os.path.join(basedir, 'exe') + if os.path.isdir(exedir): rmtree(exedir) - try: + if not os.path.isdir(exedir): os.mkdir(exedir) - except OSError: - print ("Creation of the directory {} failed" .format(exedir) ) - pigzdir = './pigz' - if os.path.isdir(pigzdir): - rmtree(pigzdir) - cmd = 'git clone https://github.com/neurolabusc/pigz' - subprocess.call(cmd, shell=True) - pigzdir = os.path.join(basedir, 'pigz', 'build') - pigzexe = os.path.join(pigzdir, 'bin', 'pigz') + ext = '' if platform.system() == 'Windows': ext = '.exe' - pigzexe = pigzexe + ext + for method in methods: os.chdir(basedir) - if os.path.isdir(pigzdir): - rmtree(pigzdir) - os.mkdir(pigzdir) - os.chdir(pigzdir) - cmd = 'cmake -DZLIB_IMPLEMENTATION=' + method + ' ..' - subprocess.call(cmd, shell=True) - cmd = 'make' + + pthreads4wdir = os.path.join(basedir, 'pthreads4w') + if rebuild or not os.path.exists('pthreads4w') and platform.system() == 'Windows': + cmd = 'git clone https://github.com/jwinarske/pthreads4w' + subprocess.call(cmd, shell=True) + + zlibname = 'zlib-{0}'.format(method['name']) + if rebuild or not os.path.exists(zlibname): + if os.path.isdir(zlibname): + rmtree(zlibname) + print("Checking out zlib source code for {0}".format(method['name'])) + cmd = 'git clone {0} {1}'.format(method['repository'], zlibname) + subprocess.call(cmd, shell=True) + + pigzname = 'pigz-{0}'.format(method['name']) + if rebuild or not os.path.exists(pigzname): + if os.path.isdir(pigzname): + rmtree(pigzname) + print("Checking out pigz source code for {0}".format(method['name'])) + cmd = 'git clone https://github.com/madler/pigz {0}'.format(pigzname) + subprocess.call(cmd, shell=True) + + os.chdir(zlibname) + if method['branch']: + cmd = 'git checkout {0}'.format(method['branch']) + subprocess.call(cmd, shell=True) + + pigzdir = os.path.join(basedir, pigzname) + copy_tree(os.path.join(basedir, 'pigz'), pigzdir) + builddir = os.path.join(pigzdir, 'build') + if rebuild or not os.path.exists(builddir): + if os.path.isdir(builddir): + rmtree(builddir) + os.mkdir(builddir) + + os.chdir(builddir) + + cmd = 'cmake .. -DZLIB_ROOT:PATH=../{0} -DZLIB_COMPAT=ON -DBUILD_SHARED_LIBS=OFF'.format(zlibname) if platform.system() == 'Windows': - cmd = 'cmake --build . --config Release' + cmd += ' -DPTHREADS4W_ROOT:PATH=../pthreads4w' + subprocess.call(cmd, shell=True) + + cmd = 'cmake --build . --config Release' subprocess.call(cmd, shell=True) - outnm = os.path.join(exedir, 'pigz' + method + ext) - print (pigzexe + '->' + outnm) + + pigzexe = os.path.join(builddir, 'bin', 'pigz' + ext) + if not os.path.exists(pigzexe): + pigzexe = os.path.join(builddir, 'pigz' + ext) + if not os.path.exists(pigzexe): + pigzexe = os.path.join(builddir, 'Release', 'pigz' + ext) + + outnm = os.path.join(exedir, pigzname + ext) shutil.move(pigzexe, outnm) + print(pigzexe + '->' + outnm) if __name__ == '__main__': """compile variants of pigz and sample compression corpus""" - install_neuro_corpus() - install_silesia_corpus() - compile_pigz() + if args.rebuild: + install_neuro_corpus() + install_silesia_corpus() + + compile_pigz(args.rebuild) diff --git a/pigz/CMakeLists.txt b/pigz/CMakeLists.txt new file mode 100644 index 0000000..8cd36b3 --- /dev/null +++ b/pigz/CMakeLists.txt @@ -0,0 +1,67 @@ +cmake_minimum_required(VERSION 3.0.2) + +project(pigz C) + +if(${CMAKE_C_COMPILER_ID} STREQUAL "Clang") + # using Clang + add_definitions(-fno-caret-diagnostics) +elseif(${CMAKE_C_COMPILER_ID} STREQUAL "GNU") + # using GCC + if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 4.5.0) + add_definitions(-Wno-unused-result) + endif() + if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 4.8.0) + add_definitions(-fno-diagnostics-show-caret) + endif() +elseif(MSVC) + # using Visual Studio + add_definitions(-D_CRT_SECURE_NO_DEPRECATE) +endif() + +set(PIGZ_SRCS + pigz.c + yarn.c + try.c) + +add_executable(${PROJECT_NAME} ${PIGZ_SRCS}) +add_definitions(-DNOZOPFLI) + +if(WIN32) + add_definitions(-D_TIMESPEC_DEFINED) + + if(NOT DEFINED PTHREADS4W_ROOT) + message(FATAL "Missing pthreads4w root directory") + endif() + + set(CLEANUP_STYLE VC) + set(PTHREADS4W_VERSION 3) + + add_subdirectory(${PTHREADS4W_ROOT} ${PTHREADS4W_ROOT} EXCLUDE_FROM_ALL) + target_link_libraries(${PROJECT_NAME} pthreadVC3) + target_include_directories(${PROJECT_NAME} PRIVATE win ${PTHREADS4W_ROOT}) +else() + find_package(Threads REQUIRED) + target_link_libraries(${PROJECT_NAME} Threads::Threads) + if(NOT APPLE) + target_link_libraries(${PROJECT_NAME} m) + endif() +endif() + +if(NOT DEFINED ZLIB_ROOT) + find_package(Zlib REQUIRED) +endif() + +add_subdirectory(${ZLIB_ROOT} ${CMAKE_CURRENT_BINARY_DIR}/zlib EXCLUDE_FROM_ALL) + +if(TARGET zlibstatic) + set(ZLIB_TARGET zlibstatic) +else() + set(ZLIB_TARGET zlib) +endif() + +target_include_directories(${PROJECT_NAME} PRIVATE ${ZLIB_ROOT} ${CMAKE_CURRENT_BINARY_DIR}/zlib) +target_link_libraries(${PROJECT_NAME} ${ZLIB_TARGET}) + +if(NOT SKIP_INSTALL_BINARIES AND NOT SKIP_INSTALL_ALL) + install(TARGETS ${PROJECT_NAME} DESTINATION "bin") +endif() \ No newline at end of file diff --git a/pigz/win/dirent.h b/pigz/win/dirent.h new file mode 100644 index 0000000..f7a46da --- /dev/null +++ b/pigz/win/dirent.h @@ -0,0 +1,1160 @@ +/* + * Dirent interface for Microsoft Visual Studio + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#ifndef DIRENT_H +#define DIRENT_H + +/* Hide warnings about unreferenced local functions */ +#if defined(__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +#elif defined(_MSC_VER) +# pragma warning(disable:4505) +#elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +#endif + +/* + * Include windows.h without Windows Sockets 1.1 to prevent conflicts with + * Windows Sockets 2.0. + */ +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Indicates that d_type field is available in dirent structure */ +#define _DIRENT_HAVE_D_TYPE + +/* Indicates that d_namlen field is available in dirent structure */ +#define _DIRENT_HAVE_D_NAMLEN + +/* Entries missing from MSVC 6.0 */ +#if !defined(FILE_ATTRIBUTE_DEVICE) +# define FILE_ATTRIBUTE_DEVICE 0x40 +#endif + +/* File type and permission flags for stat(), general mask */ +#if !defined(S_IFMT) +# define S_IFMT _S_IFMT +#endif + +/* Directory bit */ +#if !defined(S_IFDIR) +# define S_IFDIR _S_IFDIR +#endif + +/* Character device bit */ +#if !defined(S_IFCHR) +# define S_IFCHR _S_IFCHR +#endif + +/* Pipe bit */ +#if !defined(S_IFFIFO) +# define S_IFFIFO _S_IFFIFO +#endif + +/* Regular file bit */ +#if !defined(S_IFREG) +# define S_IFREG _S_IFREG +#endif + +/* Read permission */ +#if !defined(S_IREAD) +# define S_IREAD _S_IREAD +#endif + +/* Write permission */ +#if !defined(S_IWRITE) +# define S_IWRITE _S_IWRITE +#endif + +/* Execute permission */ +#if !defined(S_IEXEC) +# define S_IEXEC _S_IEXEC +#endif + +/* Pipe */ +#if !defined(S_IFIFO) +# define S_IFIFO _S_IFIFO +#endif + +/* Block device */ +#if !defined(S_IFBLK) +# define S_IFBLK 0 +#endif + +/* Link */ +#if !defined(S_IFLNK) +# define S_IFLNK 0 +#endif + +/* Socket */ +#if !defined(S_IFSOCK) +# define S_IFSOCK 0 +#endif + +/* Read user permission */ +#if !defined(S_IRUSR) +# define S_IRUSR S_IREAD +#endif + +/* Write user permission */ +#if !defined(S_IWUSR) +# define S_IWUSR S_IWRITE +#endif + +/* Execute user permission */ +#if !defined(S_IXUSR) +# define S_IXUSR 0 +#endif + +/* Read group permission */ +#if !defined(S_IRGRP) +# define S_IRGRP 0 +#endif + +/* Write group permission */ +#if !defined(S_IWGRP) +# define S_IWGRP 0 +#endif + +/* Execute group permission */ +#if !defined(S_IXGRP) +# define S_IXGRP 0 +#endif + +/* Read others permission */ +#if !defined(S_IROTH) +# define S_IROTH 0 +#endif + +/* Write others permission */ +#if !defined(S_IWOTH) +# define S_IWOTH 0 +#endif + +/* Execute others permission */ +#if !defined(S_IXOTH) +# define S_IXOTH 0 +#endif + +/* Maximum length of file name */ +#if !defined(PATH_MAX) +# define PATH_MAX MAX_PATH +#endif +#if !defined(FILENAME_MAX) +# define FILENAME_MAX MAX_PATH +#endif +#if !defined(NAME_MAX) +# define NAME_MAX FILENAME_MAX +#endif + +/* File type flags for d_type */ +#define DT_UNKNOWN 0 +#define DT_REG S_IFREG +#define DT_DIR S_IFDIR +#define DT_FIFO S_IFIFO +#define DT_SOCK S_IFSOCK +#define DT_CHR S_IFCHR +#define DT_BLK S_IFBLK +#define DT_LNK S_IFLNK + +/* Macros for converting between st_mode and d_type */ +#define IFTODT(mode) ((mode) & S_IFMT) +#define DTTOIF(type) (type) + +/* + * File type macros. Note that block devices, sockets and links cannot be + * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are + * only defined for compatibility. These macros should always return false + * on Windows. + */ +#if !defined(S_ISFIFO) +# define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO) +#endif +#if !defined(S_ISDIR) +# define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#endif +#if !defined(S_ISREG) +# define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#endif +#if !defined(S_ISLNK) +# define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) +#endif +#if !defined(S_ISSOCK) +# define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) +#endif +#if !defined(S_ISCHR) +# define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) +#endif +#if !defined(S_ISBLK) +# define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) +#endif + +/* Return the exact length of the file name without zero terminator */ +#define _D_EXACT_NAMLEN(p) ((p)->d_namlen) + +/* Return the maximum size of a file name */ +#define _D_ALLOC_NAMLEN(p) ((PATH_MAX)+1) + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Wide-character version */ +struct _wdirent { + /* Always zero */ + long d_ino; + + /* File position within stream */ + long d_off; + + /* Structure size */ + unsigned short d_reclen; + + /* Length of name without \0 */ + size_t d_namlen; + + /* File type */ + int d_type; + + /* File name */ + wchar_t d_name[PATH_MAX+1]; +}; +typedef struct _wdirent _wdirent; + +struct _WDIR { + /* Current directory entry */ + struct _wdirent ent; + + /* Private file data */ + WIN32_FIND_DATAW data; + + /* True if data is valid */ + int cached; + + /* Win32 search handle */ + HANDLE handle; + + /* Initial directory name */ + wchar_t *patt; +}; +typedef struct _WDIR _WDIR; + +/* Multi-byte character version */ +struct dirent { + /* Always zero */ + long d_ino; + + /* File position within stream */ + long d_off; + + /* Structure size */ + unsigned short d_reclen; + + /* Length of name without \0 */ + size_t d_namlen; + + /* File type */ + int d_type; + + /* File name */ + char d_name[PATH_MAX+1]; +}; +typedef struct dirent dirent; + +struct DIR { + struct dirent ent; + struct _WDIR *wdirp; +}; +typedef struct DIR DIR; + + +/* Dirent functions */ +static DIR *opendir (const char *dirname); +static _WDIR *_wopendir (const wchar_t *dirname); + +static struct dirent *readdir (DIR *dirp); +static struct _wdirent *_wreaddir (_WDIR *dirp); + +static int readdir_r( + DIR *dirp, struct dirent *entry, struct dirent **result); +static int _wreaddir_r( + _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result); + +static int closedir (DIR *dirp); +static int _wclosedir (_WDIR *dirp); + +static void rewinddir (DIR* dirp); +static void _wrewinddir (_WDIR* dirp); + +static int scandir (const char *dirname, struct dirent ***namelist, + int (*filter)(const struct dirent*), + int (*compare)(const struct dirent**, const struct dirent**)); + +static int alphasort (const struct dirent **a, const struct dirent **b); + +static int versionsort (const struct dirent **a, const struct dirent **b); + + +/* For compatibility with Symbian */ +#define wdirent _wdirent +#define WDIR _WDIR +#define wopendir _wopendir +#define wreaddir _wreaddir +#define wclosedir _wclosedir +#define wrewinddir _wrewinddir + + +/* Internal utility functions */ +static WIN32_FIND_DATAW *dirent_first (_WDIR *dirp); +static WIN32_FIND_DATAW *dirent_next (_WDIR *dirp); + +static int dirent_mbstowcs_s( + size_t *pReturnValue, + wchar_t *wcstr, + size_t sizeInWords, + const char *mbstr, + size_t count); + +static int dirent_wcstombs_s( + size_t *pReturnValue, + char *mbstr, + size_t sizeInBytes, + const wchar_t *wcstr, + size_t count); + +static void dirent_set_errno (int error); + + +/* + * Open directory stream DIRNAME for read and return a pointer to the + * internal working area that is used to retrieve individual directory + * entries. + */ +static _WDIR* +_wopendir( + const wchar_t *dirname) +{ + _WDIR *dirp; + DWORD n; + wchar_t *p; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno (ENOENT); + return NULL; + } + + /* Allocate new _WDIR structure */ + dirp = (_WDIR*) malloc (sizeof (struct _WDIR)); + if (!dirp) { + return NULL; + } + + /* Reset _WDIR structure */ + dirp->handle = INVALID_HANDLE_VALUE; + dirp->patt = NULL; + dirp->cached = 0; + + /* + * Compute the length of full path plus zero terminator + * + * Note that on WinRT there's no way to convert relative paths + * into absolute paths, so just assume it is an absolute path. + */ +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + /* Desktop */ + n = GetFullPathNameW (dirname, 0, NULL, NULL); +#else + /* WinRT */ + n = wcslen (dirname); +#endif + + /* Allocate room for absolute directory name and search pattern */ + dirp->patt = (wchar_t*) malloc (sizeof (wchar_t) * n + 16); + if (dirp->patt == NULL) { + goto exit_closedir; + } + + /* + * Convert relative directory name to an absolute one. This + * allows rewinddir() to function correctly even when current + * working directory is changed between opendir() and rewinddir(). + * + * Note that on WinRT there's no way to convert relative paths + * into absolute paths, so just assume it is an absolute path. + */ +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + /* Desktop */ + n = GetFullPathNameW (dirname, n, dirp->patt, NULL); + if (n <= 0) { + goto exit_closedir; + } +#else + /* WinRT */ + wcsncpy_s (dirp->patt, n+1, dirname, n); +#endif + + /* Append search pattern \* to the directory name */ + p = dirp->patt + n; + switch (p[-1]) { + case '\\': + case '/': + case ':': + /* Directory ends in path separator, e.g. c:\temp\ */ + /*NOP*/; + break; + + default: + /* Directory name doesn't end in path separator */ + *p++ = '\\'; + } + *p++ = '*'; + *p = '\0'; + + /* Open directory stream and retrieve the first entry */ + if (!dirent_first (dirp)) { + goto exit_closedir; + } + + /* Success */ + return dirp; + + /* Failure */ +exit_closedir: + _wclosedir (dirp); + return NULL; +} + +/* + * Read next directory entry. + * + * Returns pointer to static directory entry which may be overwritten by + * subsequent calls to _wreaddir(). + */ +static struct _wdirent* +_wreaddir( + _WDIR *dirp) +{ + struct _wdirent *entry; + + /* + * Read directory entry to buffer. We can safely ignore the return value + * as entry will be set to NULL in case of error. + */ + (void) _wreaddir_r (dirp, &dirp->ent, &entry); + + /* Return pointer to statically allocated directory entry */ + return entry; +} + +/* + * Read next directory entry. + * + * Returns zero on success. If end of directory stream is reached, then sets + * result to NULL and returns zero. + */ +static int +_wreaddir_r( + _WDIR *dirp, + struct _wdirent *entry, + struct _wdirent **result) +{ + WIN32_FIND_DATAW *datap; + + /* Read next directory entry */ + datap = dirent_next (dirp); + if (datap) { + size_t n; + DWORD attr; + + /* + * Copy file name as wide-character string. If the file name is too + * long to fit in to the destination buffer, then truncate file name + * to PATH_MAX characters and zero-terminate the buffer. + */ + n = 0; + while (n < PATH_MAX && datap->cFileName[n] != 0) { + entry->d_name[n] = datap->cFileName[n]; + n++; + } + entry->d_name[n] = 0; + + /* Length of file name excluding zero terminator */ + entry->d_namlen = n; + + /* File type */ + attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + entry->d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + entry->d_type = DT_DIR; + } else { + entry->d_type = DT_REG; + } + + /* Reset dummy fields */ + entry->d_ino = 0; + entry->d_off = 0; + entry->d_reclen = sizeof (struct _wdirent); + + /* Set result address */ + *result = entry; + + } else { + + /* Return NULL to indicate end of directory */ + *result = NULL; + + } + + return /*OK*/0; +} + +/* + * Close directory stream opened by opendir() function. This invalidates the + * DIR structure as well as any directory entry read previously by + * _wreaddir(). + */ +static int +_wclosedir( + _WDIR *dirp) +{ + int ok; + if (dirp) { + + /* Release search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->handle); + } + + /* Release search pattern */ + free (dirp->patt); + + /* Release directory structure */ + free (dirp); + ok = /*success*/0; + + } else { + + /* Invalid directory stream */ + dirent_set_errno (EBADF); + ok = /*failure*/-1; + + } + return ok; +} + +/* + * Rewind directory stream such that _wreaddir() returns the very first + * file name again. + */ +static void +_wrewinddir( + _WDIR* dirp) +{ + if (dirp) { + /* Release existing search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->handle); + } + + /* Open new search handle */ + dirent_first (dirp); + } +} + +/* Get first directory entry (internal) */ +static WIN32_FIND_DATAW* +dirent_first( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *datap; + DWORD error; + + /* Open directory and retrieve the first entry */ + dirp->handle = FindFirstFileExW( + dirp->patt, FindExInfoStandard, &dirp->data, + FindExSearchNameMatch, NULL, 0); + if (dirp->handle != INVALID_HANDLE_VALUE) { + + /* a directory entry is now waiting in memory */ + datap = &dirp->data; + dirp->cached = 1; + + } else { + + /* Failed to open directory: no directory entry in memory */ + dirp->cached = 0; + datap = NULL; + + /* Set error code */ + error = GetLastError (); + switch (error) { + case ERROR_ACCESS_DENIED: + /* No read access to directory */ + dirent_set_errno (EACCES); + break; + + case ERROR_DIRECTORY: + /* Directory name is invalid */ + dirent_set_errno (ENOTDIR); + break; + + case ERROR_PATH_NOT_FOUND: + default: + /* Cannot find the file */ + dirent_set_errno (ENOENT); + } + + } + return datap; +} + +/* + * Get next directory entry (internal). + * + * Returns + */ +static WIN32_FIND_DATAW* +dirent_next( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *p; + + /* Get next directory entry */ + if (dirp->cached != 0) { + + /* A valid directory entry already in memory */ + p = &dirp->data; + dirp->cached = 0; + + } else if (dirp->handle != INVALID_HANDLE_VALUE) { + + /* Get the next directory entry from stream */ + if (FindNextFileW (dirp->handle, &dirp->data) != FALSE) { + /* Got a file */ + p = &dirp->data; + } else { + /* The very last entry has been processed or an error occurred */ + FindClose (dirp->handle); + dirp->handle = INVALID_HANDLE_VALUE; + p = NULL; + } + + } else { + + /* End of directory stream reached */ + p = NULL; + + } + + return p; +} + +/* + * Open directory stream using plain old C-string. + */ +static DIR* +opendir( + const char *dirname) +{ + struct DIR *dirp; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno (ENOENT); + return NULL; + } + + /* Allocate memory for DIR structure */ + dirp = (DIR*) malloc (sizeof (struct DIR)); + if (!dirp) { + return NULL; + } + { + int error; + wchar_t wname[PATH_MAX + 1]; + size_t n; + + /* Convert directory name to wide-character string */ + error = dirent_mbstowcs_s( + &n, wname, PATH_MAX + 1, dirname, PATH_MAX + 1); + if (error) { + /* + * Cannot convert file name to wide-character string. This + * occurs if the string contains invalid multi-byte sequences or + * the output buffer is too small to contain the resulting + * string. + */ + goto exit_free; + } + + + /* Open directory stream using wide-character name */ + dirp->wdirp = _wopendir (wname); + if (!dirp->wdirp) { + goto exit_free; + } + + } + + /* Success */ + return dirp; + + /* Failure */ +exit_free: + free (dirp); + return NULL; +} + +/* + * Read next directory entry. + */ +static struct dirent* +readdir( + DIR *dirp) +{ + struct dirent *entry; + + /* + * Read directory entry to buffer. We can safely ignore the return value + * as entry will be set to NULL in case of error. + */ + (void) readdir_r (dirp, &dirp->ent, &entry); + + /* Return pointer to statically allocated directory entry */ + return entry; +} + +/* + * Read next directory entry into called-allocated buffer. + * + * Returns zero on success. If the end of directory stream is reached, then + * sets result to NULL and returns zero. + */ +static int +readdir_r( + DIR *dirp, + struct dirent *entry, + struct dirent **result) +{ + WIN32_FIND_DATAW *datap; + + /* Read next directory entry */ + datap = dirent_next (dirp->wdirp); + if (datap) { + size_t n; + int error; + + /* Attempt to convert file name to multi-byte string */ + error = dirent_wcstombs_s( + &n, entry->d_name, PATH_MAX + 1, datap->cFileName, PATH_MAX + 1); + + /* + * If the file name cannot be represented by a multi-byte string, + * then attempt to use old 8+3 file name. This allows traditional + * Unix-code to access some file names despite of unicode + * characters, although file names may seem unfamiliar to the user. + * + * Be ware that the code below cannot come up with a short file + * name unless the file system provides one. At least + * VirtualBox shared folders fail to do this. + */ + if (error && datap->cAlternateFileName[0] != '\0') { + error = dirent_wcstombs_s( + &n, entry->d_name, PATH_MAX + 1, + datap->cAlternateFileName, PATH_MAX + 1); + } + + if (!error) { + DWORD attr; + + /* Length of file name excluding zero terminator */ + entry->d_namlen = n - 1; + + /* File attributes */ + attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + entry->d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + entry->d_type = DT_DIR; + } else { + entry->d_type = DT_REG; + } + + /* Reset dummy fields */ + entry->d_ino = 0; + entry->d_off = 0; + entry->d_reclen = sizeof (struct dirent); + + } else { + + /* + * Cannot convert file name to multi-byte string so construct + * an erroneous directory entry and return that. Note that + * we cannot return NULL as that would stop the processing + * of directory entries completely. + */ + entry->d_name[0] = '?'; + entry->d_name[1] = '\0'; + entry->d_namlen = 1; + entry->d_type = DT_UNKNOWN; + entry->d_ino = 0; + entry->d_off = -1; + entry->d_reclen = 0; + + } + + /* Return pointer to directory entry */ + *result = entry; + + } else { + + /* No more directory entries */ + *result = NULL; + + } + + return /*OK*/0; +} + +/* + * Close directory stream. + */ +static int +closedir( + DIR *dirp) +{ + int ok; + if (dirp) { + + /* Close wide-character directory stream */ + ok = _wclosedir (dirp->wdirp); + dirp->wdirp = NULL; + + /* Release multi-byte character version */ + free (dirp); + + } else { + + /* Invalid directory stream */ + dirent_set_errno (EBADF); + ok = /*failure*/-1; + + } + return ok; +} + +/* + * Rewind directory stream to beginning. + */ +static void +rewinddir( + DIR* dirp) +{ + /* Rewind wide-character string directory stream */ + _wrewinddir (dirp->wdirp); +} + +/* + * Scan directory for entries. + */ +static int +scandir( + const char *dirname, + struct dirent ***namelist, + int (*filter)(const struct dirent*), + int (*compare)(const struct dirent**, const struct dirent**)) +{ + struct dirent **files = NULL; + size_t size = 0; + size_t allocated = 0; + const size_t init_size = 1; + DIR *dir = NULL; + struct dirent *entry; + struct dirent *tmp = NULL; + size_t i; + int result = 0; + + /* Open directory stream */ + dir = opendir (dirname); + if (dir) { + + /* Read directory entries to memory */ + while (1) { + + /* Enlarge pointer table to make room for another pointer */ + if (size >= allocated) { + void *p; + size_t num_entries; + + /* Compute number of entries in the enlarged pointer table */ + if (size < init_size) { + /* Allocate initial pointer table */ + num_entries = init_size; + } else { + /* Double the size */ + num_entries = size * 2; + } + + /* Allocate first pointer table or enlarge existing table */ + p = realloc (files, sizeof (void*) * num_entries); + if (p != NULL) { + /* Got the memory */ + files = (dirent**) p; + allocated = num_entries; + } else { + /* Out of memory */ + result = -1; + break; + } + + } + + /* Allocate room for temporary directory entry */ + if (tmp == NULL) { + tmp = (struct dirent*) malloc (sizeof (struct dirent)); + if (tmp == NULL) { + /* Cannot allocate temporary directory entry */ + result = -1; + break; + } + } + + /* Read directory entry to temporary area */ + if (readdir_r (dir, tmp, &entry) == /*OK*/0) { + + /* Did we get an entry? */ + if (entry != NULL) { + int pass; + + /* Determine whether to include the entry in result */ + if (filter) { + /* Let the filter function decide */ + pass = filter (tmp); + } else { + /* No filter function, include everything */ + pass = 1; + } + + if (pass) { + /* Store the temporary entry to pointer table */ + files[size++] = tmp; + tmp = NULL; + + /* Keep up with the number of files */ + result++; + } + + } else { + + /* + * End of directory stream reached => sort entries and + * exit. + */ + qsort (files, size, sizeof (void*), + (int (*) (const void*, const void*)) compare); + break; + + } + + } else { + /* Error reading directory entry */ + result = /*Error*/ -1; + break; + } + + } + + } else { + /* Cannot open directory */ + result = /*Error*/ -1; + } + + /* Release temporary directory entry */ + free (tmp); + + /* Release allocated memory on error */ + if (result < 0) { + for (i = 0; i < size; i++) { + free (files[i]); + } + free (files); + files = NULL; + } + + /* Close directory stream */ + if (dir) { + closedir (dir); + } + + /* Pass pointer table to caller */ + if (namelist) { + *namelist = files; + } + return result; +} + +/* Alphabetical sorting */ +static int +alphasort( + const struct dirent **a, const struct dirent **b) +{ + return strcoll ((*a)->d_name, (*b)->d_name); +} + +/* Sort versions */ +static int +versionsort( + const struct dirent **a, const struct dirent **b) +{ + /* FIXME: implement strverscmp and use that */ + return alphasort (a, b); +} + +/* Convert multi-byte string to wide character string */ +static int +dirent_mbstowcs_s( + size_t *pReturnValue, + wchar_t *wcstr, + size_t sizeInWords, + const char *mbstr, + size_t count) +{ + int error; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 or later */ + error = mbstowcs_s (pReturnValue, wcstr, sizeInWords, mbstr, count); + +#else + + /* Older Visual Studio or non-Microsoft compiler */ + size_t n; + + /* Convert to wide-character string (or count characters) */ + n = mbstowcs (wcstr, mbstr, sizeInWords); + if (!wcstr || n < count) { + + /* Zero-terminate output buffer */ + if (wcstr && sizeInWords) { + if (n >= sizeInWords) { + n = sizeInWords - 1; + } + wcstr[n] = 0; + } + + /* Length of resulting multi-byte string WITH zero terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + error = 0; + + } else { + + /* Could not convert string */ + error = 1; + + } + +#endif + return error; +} + +/* Convert wide-character string to multi-byte string */ +static int +dirent_wcstombs_s( + size_t *pReturnValue, + char *mbstr, + size_t sizeInBytes, /* max size of mbstr */ + const wchar_t *wcstr, + size_t count) +{ + int error; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 or later */ + error = wcstombs_s (pReturnValue, mbstr, sizeInBytes, wcstr, count); + +#else + + /* Older Visual Studio or non-Microsoft compiler */ + size_t n; + + /* Convert to multi-byte string (or count the number of bytes needed) */ + n = wcstombs (mbstr, wcstr, sizeInBytes); + if (!mbstr || n < count) { + + /* Zero-terminate output buffer */ + if (mbstr && sizeInBytes) { + if (n >= sizeInBytes) { + n = sizeInBytes - 1; + } + mbstr[n] = '\0'; + } + + /* Length of resulting multi-bytes string WITH zero-terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + error = 0; + + } else { + + /* Cannot convert string */ + error = 1; + + } + +#endif + return error; +} + +/* Set errno variable */ +static void +dirent_set_errno( + int error) +{ +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 and later */ + _set_errno (error); + +#else + + /* Non-Microsoft compiler or older Microsoft compiler */ + errno = error; + +#endif +} + + +#ifdef __cplusplus +} +#endif +#endif /*DIRENT_H*/ diff --git a/pigz/win/sys/time.h b/pigz/win/sys/time.h new file mode 100644 index 0000000..d029be6 --- /dev/null +++ b/pigz/win/sys/time.h @@ -0,0 +1,32 @@ +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#include +#include // portable: uint64_t MSVC: __int64 + +typedef struct timeval { + long tv_sec; + long tv_usec; +} TIMEVAL, *PTIMEVAL, *LPTIMEVAL; + +// https://stackoverflow.com/questions/10905892/equivalent-of-gettimeday-for-windows +int gettimeofday(struct timeval * tp, struct timezone * tzp) { + // Note: some broken versions only have 8 trailing zero's, the correct epoch has 9 trailing zero's + // This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC) + // until 00:00:00 January 1, 1970 + static const uint64_t EPOCH = ((uint64_t) 116444736000000000ULL); + + SYSTEMTIME system_time; + FILETIME file_time; + uint64_t time; + + GetSystemTime( &system_time ); + SystemTimeToFileTime( &system_time, &file_time ); + time = ((uint64_t)file_time.dwLowDateTime ) ; + time += ((uint64_t)file_time.dwHighDateTime) << 32; + + tp->tv_sec = (long) ((time - EPOCH) / 10000000L); + tp->tv_usec = (long) (system_time.wMilliseconds * 1000); + return 0; +} \ No newline at end of file diff --git a/pigz/win/unistd.h b/pigz/win/unistd.h new file mode 100644 index 0000000..1c7e01b --- /dev/null +++ b/pigz/win/unistd.h @@ -0,0 +1,77 @@ +/* http://ab-initio.mit.edu/octave-Faddeeva/gnulib/lib/fsync.c + Emulate fsync on platforms that lack it, primarily Windows and + cross-compilers like MinGW. + + This is derived from sqlite3 sources. + http://www.sqlite.org/cvstrac/rlog?f=sqlite/src/os_win.c + http://www.sqlite.org/copyright.html + + Written by Richard W.M. Jones + + Copyright (C) 2008-2012 Free Software Foundation, Inc. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#include +#include + +#define chown(p,o,g) 0 +#define utimes(p,t) 0 +#define lstat(p,s) stat(p,s) +#define _exit(s) exit(s) + +typedef unsigned short uint_least16_t; +typedef SSIZE_T ssize_t; + +int fsync (int fd) { + HANDLE h = (HANDLE) _get_osfhandle (fd); + DWORD err; + + if (h == INVALID_HANDLE_VALUE) + { + errno = EBADF; + return -1; + } + + if (!FlushFileBuffers (h)) + { + /* Translate some Windows errors into rough approximations of Unix + * errors. MSDN is useless as usual - in this case it doesn't + * document the full range of errors. + */ + err = GetLastError (); + switch (err) + { + case ERROR_ACCESS_DENIED: + /* For a read-only handle, fsync should succeed, even though we have + no way to sync the access-time changes. */ + return 0; + + /* eg. Trying to fsync a tty. */ + case ERROR_INVALID_HANDLE: + errno = EINVAL; + break; + + default: + errno = EIO; + } + return -1; + } + + return 0; +} \ No newline at end of file