多线程下载

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
const fs = require('fs')
const request = require('request').defaults({
        proxy: "http://127.0.0.1:10809",
        rejectunauthorized: false,
    })
    

var core = 4//几线程运行
var tsNameLen = 4 //补零长度
var len = -1
var userAgent = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4947.0 Safari/537.36'}
var delTsList = []
// console.log('开始删除')
// delTsFunc(0)  
var baseUrl = 'https://url.com'
var tsList = baseUrl+`index`
    //tsList = tsList.slice(0,-7)

//下载文件 
function download(url,filename){
    fs.readFile(__dirname + `/out/${len}.ts`,(error,res)=>{
        if(error){
            console.log(`ts-url:`,url)
            request.head(url,userAgent,(err,res,body)=>{
                if(!res){
                    console.log('下载没有成功',res);
                    return false;
                }
                if(filename != 'key'){
                    if(res.statusCode != 404){
                        request(url,userAgent).pipe(fs.createWriteStream(__dirname + `/out/${addZero(filename)}.ts`)).on('close',()=>{
                            console.log(filename,'文件下载完成',__dirname + `/out/${addZero(filename)}.ts`)
                            //count += core
                            len++
                            download(tsList+`${len}.ts`,addZero(len),len)
                        })
                    }else{
                        request(url).pipe(fs.createWriteStream(__dirname + `/out/${addZero(filename)}.ts_bak`))
                    }
                }else{
                    request(url,userAgent).pipe(fs.createWriteStream(__dirname + `/out/ts.key`)).on('close',()=>{
                        console.log(filename,'文件下载完成',__dirname + `/out/ts.key`)
                    })
                }
            })
        }else{
            //count += core
            len++
            download(tsList+`${len}.ts`,len)
        }
    })
}

//异步调用
async function init(){
    //等待删除小文件提示 否则看不到是否删除了
    for(let n = 0;n < core;n++){
        len++
        let data = await download(tsList+`${len}.ts`,len)
    }
    //下载key文件
    await download(baseUrl+`ts.key`,'key')
    return 'start'
}
//启动程序
setTimeout(()=>{
    init().then(function(data) {
        console.log('init:',data);
    });
},1000)
//补零
function addZero(n){
    //return n;
    return (Array(tsNameLen).join(0)+n).slice(-tsNameLen)
}
//删除小文件
function delTsFunc(n){
    let link = __dirname + `/out/${addZero(n)}.ts`
    
    fs.stat(link,(error,res)=>{
        if(!error){
            if(res.size<2048){
                fs.unlink(link,err=>{
                    //console.log('删除了',addZero(n),res.size)
                    delTsList.push(addZero(n))
                })
            }
            delTsFunc(++n)
        }
    })
}