Skip to content

Commit b44e138

Browse files
author
enderunix
committed
Aget
0 parents  commit b44e138

29 files changed

+1654
-0
lines changed

‎AUTHORS‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Coder & Maintainer:
2+
-------------------
3+
4+
Murat Balaban <[email protected]>
5+
6+
For questions and comments, and also bug reports,
7+
don't hesitate to mail [email protected]
8+
9+
http://www.enderunix.org/
10+
Tue May 12 08:24:56 EEST 2009

‎Aget.c‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
2+
#include <stdio.h>
3+
#include <stdlib.h>
4+
#include <string.h>
5+
#include <pthread.h>
6+
#include <unistd.h>
7+
#include <time.h>
8+
#include <fcntl.h>
9+
10+
#include <sys/stat.h>
11+
#include <sys/types.h>
12+
#include <sys/socket.h>
13+
14+
#include <netinet/in.h>
15+
16+
#include <arpa/inet.h>
17+
18+
19+
20+
#include "Head.h"
21+
#include "Aget.h"
22+
#include "Misc.h"
23+
#include "Download.h"
24+
#include "Resume.h"
25+
#include "Data.h"
26+
27+
28+
extern struct thread_data *wthread;
29+
extern struct request *req;
30+
extern int fsuggested, nthreads;
31+
extern int bwritten;
32+
extern pthread_t hthread;
33+
#include <errno.h>
34+
extern int errno;
35+
36+
37+
void get(struct request *req)
38+
{
39+
int i, ret, fd, diff_sec, nok = 0;
40+
long long soffset, foffset;
41+
char *fmt;
42+
43+
if (req->proto == PROTO_HTTP)
44+
http_head_req(req);
45+
46+
/* According to the content-length, get the
47+
* suggested number of threads to open.
48+
* if the user insists on his value, let it be that way,
49+
* use the user's value.
50+
*/
51+
ret = numofthreads(req->clength);
52+
53+
if (fsuggested == 0) {
54+
if (ret == 0)
55+
nthreads = 1;
56+
else
57+
nthreads = ret;
58+
}
59+
60+
wthread = (struct thread_data *)malloc(nthreads * sizeof(struct thread_data));
61+
62+
Log("Downloading %s (%lld bytes) from site %s(%s:%d). Number of Threads: %d",
63+
req->url, req->clength, req->host, req->ip, req->port, nthreads);
64+
65+
if (strlen(req->lfile) != 0) {
66+
if ((fd = open(req->lfile, O_CREAT | O_RDWR, S_IRWXU)) == -1) {
67+
fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno));
68+
exit(1);
69+
}
70+
71+
} else {
72+
if ((fd = open(req->file, O_CREAT | O_RDWR, S_IRWXU)) == -1) {
73+
fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno));
74+
exit(1);
75+
}
76+
}
77+
78+
if ((lseek(fd, req->clength - 1, SEEK_SET)) == -1) {
79+
fprintf(stderr, "get: couldn't lseek: %s\n", strerror(errno));
80+
exit(1);
81+
}
82+
83+
if ((write(fd, "0", 1)) == -1) {
84+
fprintf(stderr, "get: couldn't allocate space for download file: %s\n", strerror(errno));
85+
exit(1);
86+
}
87+
88+
/* Get the starting time, prepare GET format string, and start the threads */
89+
fmt = (char *)calloc(GETREQSIZ - 2, sizeof(char));
90+
time(&t_start);
91+
for (i = 0; i < nthreads; i++) {
92+
soffset = calc_offset(req->clength, i, nthreads);
93+
foffset = calc_offset(req->clength, i + 1, nthreads);
94+
wthread[i].soffset = soffset;
95+
wthread[i].foffset = (i == nthreads - 1 ? req->clength : foffset);
96+
wthread[i].sin.sin_family = AF_INET;
97+
wthread[i].sin.sin_addr.s_addr = inet_addr(req->ip);
98+
wthread[i].sin.sin_port = htons(req->port);
99+
wthread[i].fd = dup(fd);
100+
wthread[i].clength = req->clength;
101+
snprintf(fmt, GETREQSIZ, GETREQ, req->url, req->host, PROGVERSION, soffset);
102+
strncpy(wthread[i].getstr, fmt, GETREQSIZ);
103+
pthread_create(&(wthread[i].tid), NULL, http_get, &(wthread[i]));
104+
}
105+
free(fmt);
106+
107+
108+
/* Wait for all of the threads to finish...
109+
*
110+
* TODO: If a thread fails, restart that!
111+
*/
112+
for (i = 0; i < nthreads; i++) {
113+
pthread_join(wthread[i].tid, NULL);
114+
if (wthread[i].status == STAT_OK)
115+
nok++;
116+
}
117+
118+
if (nok == nthreads)
119+
pthread_cancel(hthread);
120+
else
121+
pthread_join(hthread, NULL);
122+
123+
/* Get the finish time, derive some stats */
124+
time(&t_finish);
125+
if ((diff_sec = t_finish - t_start) == 0)
126+
diff_sec = 1; /* Avoid division by zero */
127+
128+
Log("Download completed, job completed in %d seconds. (%lld Kb/sec)",
129+
diff_sec, (req->clength / diff_sec) / 1024);
130+
Log("Shutting down...");
131+
close(fd);
132+
}
133+
134+
135+
void resume_get(struct hist_data *h)
136+
{
137+
int i, fd, diff_sec, nok = 0;
138+
char *fmt;
139+
140+
nthreads = h->nthreads;
141+
142+
fmt = (char *)calloc(GETREQSIZ - 2, sizeof(char));
143+
144+
wthread = (struct thread_data *)malloc(nthreads * sizeof(struct thread_data));
145+
memcpy(req, &h->req, sizeof(struct request));
146+
memcpy(wthread, h->wthread, sizeof(struct thread_data) * nthreads);
147+
148+
Log("Resuming download %s (%lld bytes) from site %s(%s:%d). Number of Threads: %d",
149+
req->url, req->clength, req->host, req->ip, req->port, nthreads);
150+
151+
if (strlen(req->lfile) != 0) {
152+
if ((fd = open(req->lfile, O_RDWR, S_IRWXU)) == -1) {
153+
fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno));
154+
exit(1);
155+
}
156+
157+
} else {
158+
if ((fd = open(req->file, O_RDWR, S_IRWXU)) == -1) {
159+
fprintf(stderr, "get: cannot open file %s for writing: %s\n", req->lfile, strerror(errno));
160+
exit(1);
161+
}
162+
}
163+
164+
time(&t_start);
165+
166+
167+
#ifdef DEBUG
168+
for (i = 0; i < nthreads; i++)
169+
printf("Start: %lld, Finish: %lld, Offset: %lld, Diff: %lld\n",
170+
wthread[i].soffset,
171+
wthread[i].foffset,
172+
wthread[i].offset,
173+
wthread[i].offset - wthread[i].soffset);
174+
#endif
175+
176+
for (i = 0; i < nthreads; i++) {
177+
wthread[i].soffset = wthread[i].offset;
178+
wthread[i].fd = dup(fd);
179+
snprintf(fmt, GETREQSIZ, GETREQ, req->url, req->host, PROGVERSION, wthread[i].offset);
180+
strncpy(wthread[i].getstr, fmt, GETREQSIZ);
181+
pthread_create(&(wthread[i].tid), NULL, http_get, &(wthread[i]));
182+
}
183+
184+
for (i = 0; i < nthreads; i++)
185+
pthread_join(wthread[i].tid, NULL);
186+
187+
for (i = 0; i < nthreads; i++) {
188+
pthread_join(wthread[i].tid, NULL);
189+
if (wthread[i].status == STAT_OK)
190+
nok++;
191+
}
192+
193+
if (nok == nthreads)
194+
pthread_cancel(hthread);
195+
else
196+
pthread_join(hthread, NULL);
197+
198+
199+
200+
time(&t_finish);
201+
if ((diff_sec = t_finish - t_start) == 0)
202+
diff_sec = 1; /* Avoid division by zero */
203+
204+
Log("Download completed, job completed in %d seconds. (%lld Kb/sec)",
205+
diff_sec, ((req->clength - h->bwritten) / diff_sec) / 1024);
206+
Log("Shutting down...");
207+
close(fd);
208+
}

‎Aget.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#ifndef AGET_H
2+
#define AGET_H
3+
4+
#include "Data.h"
5+
#include "Resume.h"
6+
7+
void get(struct request *);
8+
void resume_get(struct hist_data *);
9+
10+
#endif

‎COPYING‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
2+
Copyright (c) 2002 , Murat Balaban <murat at EnderUNIX dot ORG>
3+
4+
All rights reserved.
5+
6+
Redistribution and use in source and binary
7+
forms, with or without modification, are
8+
permitted provided that the following
9+
conditions are met:
10+
11+
1. Redistributions of source code must
12+
retain the above copyright notice,
13+
this list of conditions and the
14+
following disclaimer.
15+
16+
2. Redistributions in binary form must
17+
reproduce the above copyright notice,
18+
this list of conditions and the
19+
following disclaimer in the
20+
documentation and/or other materials
21+
provided with the distribution.
22+
23+
3. Neither the name of the EnderUNIX Team
24+
nor the names of its contributors may
25+
be used to endorse or promote products
26+
derived from this software without
27+
specific prior written permission.
28+
29+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT
30+
HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
31+
EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
32+
BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
33+
OF MERCHANTABILITY AND FITNESS FOR A
34+
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
35+
EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
36+
LIABLE FOR ANY DIRECT, INDIRECT,
37+
INCIDENTAL, SPECIAL, EXEMPLARY, OR
38+
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
40+
OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
41+
OR BUSINESS INTERRUPTION) HOWEVER CAUSED
42+
AND ON ANY THEORY OF LIABILITY, WHETHER IN
43+
CONTRACT, STRICT LIABILITY, OR TORT
44+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
45+
IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
46+
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
47+
DAMAGE.
48+
49+
50+
Tue May 12 08:43:20 EEST 2009

‎ChangeLog‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
EnderUNIX Aget ChangeLog
2+
3+
* Tue May 12 08:25:29 EEST 2009
4+
Version 0.4.1 adds/changes the following:
5+
- Fixed errno problem on newer glibc's
6+
- Large file support (files bigger than 2 GB)
7+
(Thanks to Siwei Liu of Sun Microsystems)
8+
9+
* Fri Nov 22 07:44:20 EET 2002
10+
Version 0.4 adds/changes the following:
11+
- Fixed bugs in Resume code.
12+
13+
- Fixed miscalculations in the progress indicator
14+
15+
* Fri Sep 20 18:11:11 EEST 2002
16+
Version 0.39 adds/changes the following:
17+
- Hafiye is not 100 % POSIX compliant now.
18+
pread/pwrite functions are now used to
19+
simultaneously write to the local file.
20+
21+
- Because of pwrite, the local file is not
22+
assembled at the end now, it's assembled
23+
at the same time. This speeds up large
24+
downloads; since previously lots of I/O was used
25+
to assemble the file. This is now decreased by
26+
at least fifty percent.
27+
28+
- Solaris bug fixed.
29+
30+
- Resume support.
31+
32+
- Some more options:
33+
-p [port number to connect to]
34+
-l [local file name, if different from the remote file]
35+
-f [force use of suggested number of threads]
36+
37+
- A progress indicator has been added.
38+
39+
- Code has been re-arranged. Since aget starts
40+
to become larger, that was a need.
41+
42+
- A README-Developer has been added for
43+
developers who want to contribute to aget.
44+
45+
46+
* Fri May 3 23:25:08 EEST 2002
47+
Version 0.2p1 adds/changes the following:
48+
- getopt() problem is now solved,
49+
The program should really run on Posix systems.
50+
I've tested it on Linux, FreeBSD ans Solaris.
51+
52+
- If no thread number is supplied, it is defaulted to 1.
53+
54+
- A Solaris Makefile is added to the tarball.
55+
56+
* Wed May 1 18:15:32 EEST 2002
57+
Version 0.2 adds/changes the following:
58+
- File is now created seperately in each thread
59+
and later, it is assembled.
60+
Thus, some overhead from locking and unlocking of
61+
the mutex has been decreased.
62+
63+
64+
* Wed May 1 10:43:44 EEST 2002
65+
First Public Release: 0.1

‎Data.h‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#ifndef DATA_H
2+
#define DATA_H
3+
4+
#include <pthread.h>
5+
#include <netinet/in.h>
6+
7+
#include "Defs.h"
8+
9+
typedef struct request {
10+
char host[MAXHOSTSIZ]; /* Remote host */
11+
char url[MAXURLSIZ]; /* URL */
12+
char file[MAXBUFSIZ]; /* file name */
13+
char lfile[MAXBUFSIZ]; /* if local file name is specified */
14+
char ip[MAXIPSIZ]; /* Remote IP */
15+
char username[MAXBUFSIZ];
16+
char password[MAXBUFSIZ];
17+
int port;
18+
long long clength; /* Content-length */
19+
unsigned char proto; /* Protocol */
20+
} request;
21+
22+
typedef struct thread_data {
23+
struct sockaddr_in sin;
24+
char getstr[GETREQSIZ];
25+
long long soffset; /* Start offset */
26+
long long foffset; /* Finish offset */
27+
long long offset; /* Current Offset */
28+
long long clength; /* Content Length */
29+
int fd;
30+
pthread_t tid; /* Thread ID */
31+
unsigned char status; /* thread exit status */
32+
} thread_data;
33+
34+
#endif

0 commit comments

Comments
 (0)