省级精品课程网站,php简易企业网站源码,免费建站自助建站网站建设教程网站建设教程,网站app有哪些功能论文
地址
概述
该论文提出了一种名为LISA的层重要性采样优化算法#xff0c;旨在解决大规模语言模型训练中的内存瓶颈问题。其主要内容和贡献包括#xff1a;
通过分析LoRA训练中各层权重范数的分布#xff0c;发现了权重更新在底层和顶层更为集中的现象#xff0c;揭…论文
地址
概述
该论文提出了一种名为LISA的层重要性采样优化算法旨在解决大规模语言模型训练中的内存瓶颈问题。其主要内容和贡献包括
通过分析LoRA训练中各层权重范数的分布发现了权重更新在底层和顶层更为集中的现象揭示了不同层在训练中的不同重要性。基于上述发现提出了一种简单的策略即根据权重范数分布随机冻结大部分中间层只优化对模型影响较大的底层和顶层从而减少训练时的内存消耗。引入了一种名为LISA的优化算法利用重要性采样思想根据各层的权重范数分布来设置冻结层的概率实现高效训练。实验结果表明LISA算法在多种下游微调和域特定微调任务中均优于LoRA和全参数训练展现了其优越性。LISA算法可扩展到超过70亿参数的大规模模型同时具有与LoRA相似的内存消耗为解决大规模语言模型训练中的内存瓶颈问题提供了一个有效解决方案。 综上所述该论文通过对LoRA训练的统计分析提出了一种简单高效的层重要性采样优化算法LISA在多种语言模型微调任务中展现了优越性能为大规模语言模型的训练提供了一种高效解决方案。
代码
import mathimport numpy as np
import paddle
import paddle.nn as nn
import paddle.nn.functional as Fclass MaskMultiHeadAttention(nn.Layer):def __init__(self, hidden_size, num_heads):super(MaskMultiHeadAttention, self).__init__()assert hidden_size % num_heads 0, Hidden size must be divisible by the number of heads.self.hidden_size hidden_sizeself.num_heads num_headsself.head_size hidden_size // num_heads# Query, Key, Value matricesself.query_linear nn.Linear(hidden_size, hidden_size, bias_attrFalse)self.value_linear nn.Linear(hidden_size, hidden_size, bias_attrFalse)# Output matrixself.output_linear nn.Linear(hidden_size, hidden_size)self.sin_em, self.cos_em sinusoidal_position_embedding(50000, self.head_size)def forward(self, x, stateNone, seq_lenNone):batch_size x.shape[0]# Compute Query, Key, Value for all heads in parallelquery self.query_linear(x).reshape([batch_size, -1, self.num_heads, self.head_size]).transpose([0, 2, 1, 3])value self.value_linear(x).reshape([batch_size, -1, self.num_heads, self.head_size]).transpose([0, 2, 1, 3])query rope(query, self.sin_em, self.cos_em, seq_len)# Compute attention scoresscores (F.relu(query) F.relu(-query)) / (self.head_size ** 0.5 0.000000000001)if state is None:state 0state paddle.cumsum(scores, -2) 0.0000000000000001 statescores scores / statestate state[:, :, -1:]out scores * value# Concatenate and transform to get the final outputout out.transpose([0, 2, 1, 3]).reshape([batch_size, -1, self.hidden_size])out self.output_linear(out)return out, stateclass FeedForward(nn.Layer):def __init__(self, hidden_size):super(FeedForward, self).__init__()self.ffn1 nn.Linear(hidden_size, hidden_size // 2)self.ffn2 nn.Linear(hidden_size // 2, hidden_size)self.relu nn.ReLU()def forward(self, x):x self.ffn1(x)x self.relu(x)x self.ffn2(x)return xclass GPTDecoderLayer(nn.Layer):def __init__(self, hidden_size, num_heads):super(GPTDecoderLayer, self).__init__()self.self_attention MaskMultiHeadAttention(hidden_size, num_heads)self.ffn FeedForward(hidden_size)self.norm1 nn.LayerNorm(hidden_size)def forward(self, x, stateNone, seq_lenNone):x1, state self.self_attention(x, state, seq_lenNone) # Self-Attention with residual connectionx x1 xx self.norm1(x)x self.ffn(x) x # Feed-Forward with residual connectionx self.norm1(x)return x, stateclass PositionalEncoding(nn.Layer):def __init__(self, d_model, max_len5000):super(PositionalEncoding, self).__init__()# Create a long enough Paddle array to hold position encodings for the maximum sequence lengthposition paddle.arange(max_len).unsqueeze(1).astype(float32)# Create a constant pe matrix with the same size as the embedding matrixdiv_term paddle.exp(paddle.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))pe paddle.zeros([max_len, d_model])pe[:, 0::2] paddle.sin(position * div_term)pe[:, 1::2] paddle.cos(position * div_term)self.pe pe.unsqueeze(0) # Shape: [1, max_len, d_model]# Register pe as a buffer (non-trainable parameter)def forward(self, x, seq_lenNone):# x is of shape [batch_size, seq_len, d_model]if seq_len is None:seq_len x.shape[1]return x self.pe[:, :seq_len, :]else:return x self.pe[:, seq_len - 1:seq_len, :]# %%def sinusoidal_position_embedding(max_len, output_dim):# (max_len, 1)position paddle.arange(0, max_len, dtypefloat32).unsqueeze(-1)# (output_dim//2)ids paddle.arange(0, output_dim // 2, dtypefloat32) # 即公式里的i, i的范围是 [0,d/2]theta 10000 ** (-2 * ids / output_dim)# (max_len, output_dim//2)embeddings position * theta # 即公式里的pos / (10000^(2i/d))sin_embeddings paddle.sin(embeddings)cos_embeddings paddle.cos(embeddings)return sin_embeddings, cos_embeddingsdef rope(q, sin_em, cos_em, seq_lenNone):if seq_len is None:sin_em sin_em[:q.shape[2]]cos_em cos_em[:q.shape[2]]else:sin_em sin_em[seq_len - 1:seq_len]cos_em cos_em[seq_len - 1:seq_len]q1 q.reshape([q.shape[0], q.shape[1], q.shape[2], -1, 2])[..., 1]q2 q.reshape([q.shape[0], q.shape[1], q.shape[2], -1, 2])[..., 0]# 奇数负值*sin_em偶数正值*cos_em 奇数正值*cos_em偶数正值*sin_emq3 paddle.stack([-q1 * sin_em q2 * cos_em, q1 * cos_em q2 * sin_em], -1)q q3.reshape(q.shape) # reshape后就是正负交替了return qclass GPT(nn.Layer):def __init__(self, vocab_size, hidden_size, num_heads, num_layers):super(GPT, self).__init__()self.embedding nn.Embedding(vocab_size, hidden_size)self.decoder_layers nn.LayerList([GPTDecoderLayer(hidden_size, num_heads) for _ in range(num_layers)])self.fc nn.Linear(hidden_size, vocab_size, bias_attrFalse)def forward(self, x, stateNone, seq_lenNone):x self.embedding(x)# x self.position_embedding(x, seq_len)if state is None:state [None] * len(self.decoder_layers)i 0for decoder_layer in self.decoder_layers:x, state[i] decoder_layer(x, state[i], seq_lenNone)i 1out self.fc(x)return out, statedef check_rope():q paddle.randn((8, 12, 10, 32))sin_em, cos_em sinusoidal_position_embedding(50000, 32)rope(q, sin_em, cos_em)def check_mask_multi_head_attention():x paddle.randn([4, 10, 20])mha MaskMultiHeadAttention(20, 5)out mha(x)def check_positional_encoding():x paddle.randn([4, 10, 20])pe PositionalEncoding(20)out pe(x)def check_gpt_decoder_layer():x paddle.randn([4, 10, 20])dl GPTDecoderLayer(20, 5)out dl(x)def check_gpt():x paddle.randint(4, 10, [4, 10])gpt GPT(10, 20, 5, 2)out gpt(x)def check_lisa_adam_w():gpt GPT(10, 20, 5, 2)# 初始化模型和超参数nl len(gpt.parameters())T 100K 10gamma 2eta0 0.001pnl gamma / np.ones(len(gpt.parameters())) * gamma / nlpnl[0] 1pnl[-1] 1criterion nn.CrossEntropyLoss()optimizer paddle.optimizer.AdamW(parametersgpt.parameters(), learning_rateeta0)# 训练模型for i in range(T // K):# 冻结所有层for l in gpt.parameters():l.trainable False# 激活层采样for l, p_l in enumerate(pnl):print(gpt.parameters()[l].shape)if paddle.rand([1]) p_l and len(gpt.parameters()[l].shape) 1:w_lora_l paddle.norm(paddle.linalg.svd(gpt.parameters()[l])[0], p2) paddle.norm(paddle.linalg.svd(gpt.parameters()[l])[2], p2) # LoRA权重范数w_lisa_l paddle.norm(gpt.parameters()[l], p2) # LISA权重范数p_l_1 w_lora_l / w_lisa_loptimizer.set_lr(optimizer.get_lr() * p_l_1.item())# 激活该层gpt.parameters()[l].trainable True# 训练激活层for _ in range(K):x paddle.randint(4, 10, [4, 10])out gpt(x)# 输出模型参数# print(gpt.state_dict())def check_all():check_lisa_adam_w()check_rope()check_mask_multi_head_attention()check_positional_encoding()check_gpt_decoder_layer()check_gpt()if __name__ __main__:# 检查所有模块check_all()
感谢
李清舟至味人间沐曦baba春雪芳菲DataMastermindArtemisMAI皇城根奶爸一个小老虎遥远的瞧飞黄腾踏去小狐狸M一见导师就变乖JamesHarden_zibuyu9金城七verwandte鄢希不是糖带带小猪頭laforet珍儿要有梦Alkaid.K打打印机优达学城UdacityKerstinTongxi秦老猫weixin_32592297勃拉图杜雨茜8枚伊红美兰鹤运赖智超嘉禾博研左方程2401_84036823ShirahaneSuoh三脚猫功夫猴知心大哥czsl苏远岫夜十六刘孝虎翻山狮雯雯呀鄂奎阿冯明溪刘不言孤傲雕CRomputer-罗军俞组weixin_28916013雾里元宝斗鱼直播-大司马~~钢蛋鸭各自安好吧徐辉947序雨python作业毕设Olyg012荣耀张大仙mk~~黑雾思索者JanDuo小妖戴某DEMO雨前羽街easycofe赵利兴康儿妈咪文强孙小菜鸡记录生活恐怖电影爱好者atQ4N张钧琦Lhmily0718久作无端客NetNinjaSabrina5点的太阳Quant最爱32罗翔说~~~聪明帅气的阿凯你来说我来听无心高调张大胃熙南君pink怒怒酱开东本书本轻萌小说喂今天救公主了吗糯米不是饭OurPlay弥古爱书法UEgood雪姐姐柏林墙东侧非典型普通人类Gdsw-Dpp夏北方南先生AC编程宇哥讲电影wx24e331e6d83e1b9fPixelPuzzlist伊斯特艾格weixin_34354231初小轨医脉通卢梅庆乐观观天下孔皓一块烤地瓜荷达土豆做成泥很好吃唰唰说创业muyyyi665733489631玄铁匠珞珈小山a301kaorouwoyaochi甜甜爱笑拉斯科纳夫谐波治理小陈腾岳掌门KimiOnZhihu中国计算机学会榣木invalid火猫人有石田仲政好好医生one京城风四娘rainman丹的小跟班灰色的男孩明德zhuang夏之幻想好什一王连涛云保链CIC科技娜评月华剑士张大彪罗大哥柒年tenderweixin_36462532伍世棋甘团路怂喵rise6d影子医生WIDGAF氧化三氢正离子清水33张敬禹Simon厂妹blossomzwjAI前线Elsafun爱地人konjunkturtankCYTcodgician两半24409Lucas鱼利安一两赘肉无叶建均Julojulo金利来Goldlion晋小睿(已退乎)不爱吃mao暖色Cecilia渔翁丶稀饭香江不知名前浪Kelly小后羿DigitalElusion三叔侃侃苏沉船岁月未染携程邮轮铁拳无敌双鸭山Troy爱滑雪智者也流氓拳宗师zhoudiNEU三三俩俩罗小熙尼克六六2401_83815144科技888laplaceliu白泽之水木子淇medabroad斯迈尔齿科金渡江吕驰宇jzjzjxjxj假Rrrrrrr_a喝墨水的猫武藤2401_83167347InfoMaster张奕明Cyberjkyu我为歌狂有崽了CVRunner老凯撒的钱豆蔻茶我在四平路上gonggon吃素的pig戴戴好太田秋明errommel赵少林稚一师父曰魔术狗18767873417璐寶吉田区块链Shaun阿呆酱SlyerD白酸奶暮汐颜朱庇特不是宙斯切高仔果舒花猫哥哥徘徊孤独弘世堇孙继军卤蛋鱼丸粗面张允健曹逆娘曹啊呀呀王羿秋42号车库剑使者日签君AIUX影歌小队长Marcus木头骨头石头明治奶茶味雪糕张盛锋Freya鸭鸭Davider_Wuhhhjknbbyco依JO孔祥謙温哥华小文青李大颖misshohxil青山布衣慕云飞卡休微卡陳懋兒Hannah豆腐LZ钮钴禄·缇Naoki东方安璇Paradox解忧小巫仙溯水襄陵陈昊芝徵琛开发者傑王一点寒吴光于小大隐刘军莉BossGuo果老板甲烷生产者司炉程军超綺懷weixin_40675290苏友学DaydayydayyyMiuyana吴裕彬江涛校泵八卦钩沉冯鑫怡游道人圆宗亮姐樰篱今朝有昔G僧东weixin_40819800爱怹顾汐漫依然念鼬风之木叶住颜off彩蛋王希亚MeflsBrandon邱荣城我不发文章啦北西南风林志强一颗墙上的稻草水精灵琼子来自宇宙外的未知生物周天无极十万泼皮连续失眠者CFredbottomerHyperliquidX杨良昆Shelly陳一楠其实只是平凡人就叫D吧PraiseSunMan解说柯基\x01mkq0.~VizXu红糖小糍粑墨墨张章王舜皮肤医生朱岳衡林忆酒句子院七彩极光淬过火的家鸽维林兄弟Krisinmel沈鸫驿燕仰据克新语丝漩凝陪刘耀文写五三声振研究言雨潇Francis裕文璋球球铭刻录刘权德氪老师15886991492a1072024灵境里的孤舟别人的鞋JIA-璐介敏狼狼小丑逼天下的乌鸦不一般黑分秒帧丶方可草莓味儿柠檬peyton-?这里是益达面料知识馆胖酷帅张宜辅qq5471179162将作于少监真正的小蚂蚁Piper蛋窝琥珀每人宠头条Java天骄雪麻麻ylJovi.wang淡于水人知广实战剑术路人FIM架狙只打脚秋凤梧chinhoyooweixin_53947681葵葵有问题知之者不如好之者weixin_49192623freeviltanakajohn张小璇心碎的恶魔ALEX刘天昭yutin020长腿小姑娘lambdaJisleep豆时维教育顾老师左歪阿木吃饭用大碗扒熊牛先森讲故事的妹妹emitofomemeda澜方井冉krisyoung1028苏沃洛夫韭菜后浪任博冰Bob莫尔奔高辰光MHJCR彭迅鹏xpCyberGoddess温柔对待自己尔你苟全性命于治世Bonsen摘星星的男孩芒果加柠檬About勤劳课代表xinwuji312唐风宋影Demon学长据说名字要够短耗子不偷油勋哥很忙疯而不jue大海贼青木兔撕鸡大老爷dgccfhhv相月十久绳子上的蚂蚱卫迟星空下的史诗ScriptHero熬夜编程小垃圾不知彼岸李德邻Tfiftheweixin_51792875删号怎么尕冯正华cohenLucemon瓜瓜龙左脚刹车右脚油门刘兮litterbug21Virtus杨香蕉日更阿婆主播仁杰请叫我段教授超越学姐杭州分越流风兮回雪Project东坡不改了藩依然胡天宝西门子中国ScandalRafflesiaSY-EU德易至日读课味离chunhui首席测评师王Tie塔儿Scarlett(王萍)饶龙友柳飞祥pk2017Youlink薛莹莹de洋果子肖深刻的九叔ECUSTJared不执不迷呦鱼儿weixin_28730665亿丁丁Facilidad彬者·诡道真格量化蔡千年相争何为杰长老redchina111frankie知之只weixin_36972161绥哲所谓大力小豆一小撮人老年吉米AY-onmyway兰艳知己Yang一颗青木喜迁莺MioZzh3514一躲躲乔克蜀黍丶精明败家女杨爷零悠悠笔杆abc高中物理宋老师菲比她他它阿弥陀佛蓦风星吟钭胥冉胡世博养猪编码工dyingflames酒夏品牌烧友-A郝艺益可可子姐姐教英语厉向晨13512954815Camelliashine-well日向夕阳薛玉玺吴名璞怪叔叔来了核电那些事无锡星晴侯鹏飞清峪沟的砍柴人猫咪的室友巴白比波布da北斗retffts0721小小仙女儿Moriarty无糖信息忧伤的石一农资从业者舜祎魂赵阿爷兰borninbad德格扎西次仁普罗马克走进良夜Purkialo织浪邹正限键盘音乐家米斯特JIANG鱼水鲤缘学术入门285995986森因那夫tkh831219好奇de梓洋雷米斯冯委就是雪马宇轩东海羊驼灰熊问题最优解浅盏半茗茶磅礴科技路I胡侃侃兰德水TA家住在沙漠中碧海云天97霜之韵逸萧井陌酱好不墨尔本雪球兔BinaryBard陈六六的成长笔记雕牌优质眼药水袁廷翠09050082月半羊懒小木王Jason王迷茫不再迷茫qwrrq12134理性且坚决飞清一色Dexter.YyMs雄心似铁风自林琬清刘照云卡玫瑰妈妈Amy婧张浩驰松果煲粥奶茶余香Wonder王达西芹姑娘添奇艺术伞-立辉Chau.Hui想法臃肿物语妖腿姐姐u牧心钰CassietalitzyCyberHero赵仙君CyberNinja杨浩鸣一璇往事浮烟遙朢鷰園苏式糖粥派网大话狗玩木丧志辟克匿克lkxl九酱酱酱酱酱颜镇钟群响-刘思毅黄叶欣iff-qyc亦未卜仆weixin_31092081藏雍之金吉他堞堞mydiplomacy宝贝入怀诚is黄衔芊暖徐行忠大白兔奶糖呀牛站长插座学院诚灏米佗耶目了了她姐我画的娃娃脸多问jimy樊旖斐诗人流浪汉作者小怪兽weixin_36955541住在树上的牛顿2301_80904942维权骑士徐再旭zhihuyaowan阿证证证证娃咋养伊迪亚特十弦曲觞躺着不干活fatgn兮扬李国凤龙黄天下牛奶浴花生小松ekko2401_83080457诺壹乔上官义飞没有名字233薛继续2301_80585628姜懿珊Mr浪子相依jackey-金融八度飞雪大叔默默路过Tonexpf769815576PrivateRookie蛋卷毛毛2301_81244691银光秋叶胡格DigitalDragon衡水中屑official运动与物理康复123456zggb牛开心pmbian陈fay游很多话宋玄宾林潘Zeng我的小幸运南极熊3D打印咻电联盟18336067661prettySXY官真千面娇娃酒不修m0_73127071茶未凉人已散碧微公子呱呱呱wwwwei0168520你猜-我是谁鞠猫曜冰ONES长余陆离weixin_58632235guonilp冷幽香晚点LatePost天氣真好喔2401_83010645我有烧饼你有酒吗酱呀cyhjty唐臭臭ICT杂谈甘雨火光雷幺幺懒床上的猫江啾0000花开知多少让我眼熟你北山南湖兰榕笙托卡马克之冠\x01m~~燕衔泥PPT狠茬子嘻嘻飞翔的嘉别ChuaChuaFeng牛人说baby小花朱耶伽罗李奇正邱教练学堂庄蹊南洋野人DeepWeaver材料搬砖狗叶苏培Gavin一捆稻草垛youye庄黑胖-伍拾万naif_cy一杨千里迢迢牵牛星天街踏尽公卿骨兔子同学夕风彦khaki轩然已经退乎的路人Dumb丁郭啧西请给我骨头北京永诚印刷邹益健往里卷AllFiredUp对视三秒Lrsmn-x潜水队长锋剑一族耀眼曦光小灰灰同学HermanXYZ富富设计胃泰小胃君叶修的伞卢策吾宁敏CN队长本尼桑岑做一土水丰色今口偏偏无理取闹小巧的高跟鞋mhj199011先生肉丸君宁艳玲sweetTKSJ李华晨摩羯啪啦啪啦Lilililiprettywow机智兮米粒丘罪歌2011天才娜娜lnAT姜你找不到我啦its斯弟文一米八七的大帅哥北木南烟叶伏城陈华葵xclca非典型工科生何书欣肉身君罗宏宝会写代码的好厨师Nature自然科研李首良已婚屌丝[HUC]–kshbig曹胜波Evelyn勃恶霸太球惢安科技缪缪观后感刚刚好行影旅行rrKnow旋叶芦荟mkqdarkdress我喜欢你小朋友苏安桥江思捷李月巴鱼解说柯基\x01mkq腕表天地阿文说钱西瓜苦力怕何沐沐亚大伯斯Kotori无伤大雅的骆驼默认关系站在天台上看上帝兔卡知乎商业小管家谢月生晓文说财weixin_40423078顺数人Tatasisy大地孤狼袁大岛燕枝我寄几苏瑞文精神心理何日辉机械堡垒葛艺潇hore12月之地林中白鹿黄莉娟m0_75009479宋世泊peng欧阳万里白衣战土撸铁妹思哈豆夜盏陈仁浩在韩国的敏欧尼myosinYan东南前哨番茄罐游魂2Balinda婷婷greatwallismeMaple老徐说Hi-iDPettySoul伟星啊密斯how蒋栋蔚Fei运营老高水廿八于伟豪gaojb2141129weixin_44983006简暂停一下下北美地产学堂lange清宵月明马利碳笔zLiM5不止尚行光阳果唐杰宇雾里寻声刘宏山083筛分技术weixin_28726871黄差败whygqm瓦罗兰十字军SlowdownDdddddd精灵之火Capex刘杭州Tim爱燃烧水龙敬智诚1818鎏怀瑾20250101易职嗨柯学友歹小悦filweod198902会员联盟weixin_41699549玺微の时光卫酷酷Reaper2301_80615417weixin_59061340CDKingHang2301_817860372401_825908012301_793707222301_807355422301_804710802401_82563337ashshouldgo2301_81323248xiaogang2062201_757087772301_81875574nipponofficial2301_816694152301_82229336MessyCandle丶qq_443884692301_81211889linhj20 阳和启蛰8342401_824071732302_796046262302_80430759hp125626upup293路野没有苏你好刘同学布拉达勇士2301_82091956連1172303_81635589爱吃薯条的l杀手海王u013665684kmn04Famiglistimo_01Neptune2301_82105743凯瑟琳大王2401_82478192忙着长大的丫丫2401_82497365萌萌1212|2301_81343863天盗盗2401_82491506柠檬小仙女CRiceman2401_823974062401_82484621qbkivlinChrix君有点小暖剪好头发回家weixin_40564623薇莉木楠依苏小晗马骏骁Dr.Rodneyweixin_44710965会飞的洋子成都AG超玩会老帅Mikeylivhui2301_79218897不想说话的树maggie67m0_62212436333M张卡特周晓农山簡坚强努力地活下去兔兔xuanyaoyao是我住范儿石乐天呼呼啦啦就瘸了大司马了不起周达和雷雨航清清本清诶你说什么summerrabitYE625就念Raymond袁江旭辟谣的大舌头LONG悠悠黄鹤楼鵬鵬鵬鵬丁子栩清风余欢林秀秀gCodeWhiz我鸿weixin_30453651吴寒笛蓝盐泳池1983销声真实故事计划卖猫粮的小叔元音a马渊程杜扬Seatory南燕JoInnovationGuru老先队员小xs原点GD华清阙lcct_shu谈修竹TechTinkererSeolac野蛮人柯南weixin_27727467你一直在玩儿屁师春华秋实的夏我累个擦宋朗坤Ruby437無酒郴江郑明兰心理咨询师-宋杨Twopothead护城河河长ianlesliejet弍戈林中有一只小猫猫张蛋花缪加少刷知乎多读书海边的弗兰克纪发发秋水顾盼煌煌不安myy4988Dreamer寻好孩纸kimi爱大圣米凯勒雪停时偶遇一叶春kagbElka投稿小助手橘子咚咚wanchuanlongmoxirichqq_55253983刘锋135538569000托尼郭Lybrusta291337862江泓有点小病寻尘客六号轨迹丹凤张杰王靖海Tasrue雕粱画栋胡启奋人间最好小师叔gaocegege张大有网易《了不起的中国制造》026后勤处re硕君张玮(O-o)布拉格2I6and7李军文柯尔鸭鸭龙一歌霉斯漫我爱专利bathroom火冒blhzmmf上海买房人颜究员堂堂堂堂堂主江文锴门田木LiBooks.Fan夏洛特Charlotte希茅张坤平安D先生parisbob指路明灯君菜菜美食日记乞力马扎罗的猴子安藤崇WawzyniecCaprcornliuzhiyaooooྲ552司马各晒太阳的小黑猫2301_807618952301_81284979江千里lsgn方头Pokermanmonprotocol2301_78823239weixin_574499522301_79683098好き丸子AlpHaCrackerhzhangsaMiaozerong2301_81056293m0_720415202301_81213077望月愁拿秃子的酒杯朱凤仙啥雷慧星2301_77149178Tagtax2301_81188326田哥玛丽2305_79018775云扰otx009super2301_81192457影视后期王荣林声飘扬逆光纪阿廖林诺一人一猫浪迹天涯温糯米穆迪老师一滴血的记忆好奇宝宝打脸怪谢艺馨Lucy-露西娅淡庸Manting西天取经路遥迢型爷诗无忌Luca_Luxsvipx4869Rachel瑞小秋Damon尘箬芳桦达欣欣东予薏米罗元裳麓雍黄鱼冻早鸟江仕蕾王金龙花柒咸鱼豆腐LearningandStudy幸福暗恋我冰镐供应商克莱儿儿儿quant林梦里看花a华亿司徒夜Gnocchiiii心理咨询师林博宇3菲菲andyyah明日座头鲸曼仔呀不吐槽会死星人i燕山美发央企小老头跑马溜溜Ming小然一朵云从平流层坠下何钦尧吃肉的梅小姐每天一个收货王谭陈村埃斯蓬托的篡位者俯卧撑菜鸟怪怪的Mogeko最上川灵感大王qq_48582659noah642人and人m0_64601486Yorium2301_80668056好588RyFooX2301_76448989用户昵称已存在……错误……itwebber林闽琦wxd86d5613e425e18b2301_800712682301_764036202301_810110432301_810139402301_80383138weixin_46615692微软技术分享黔台老酒坊小瓮同学灵魂机器耀怡baskicelongyang0917啃老师有伱真好罗俊(顺义PICC)刘为龙馋包包我们走葛店小学张洪雨发现者号biu~FUUFKm0_67598909南小吕m0_61393398ShAn.weixin_39505051借我一把金太刀不会飞的鹰08qq_64694880weixin_47898112qq_45292756希出崖涘20240901GALAXYLllixym0_70563695m0_742821032301_77131751MasterZerαm0_61098022qq_48962699weixin_51793392CSDN_alearner2301_80044454Cyber_D沈聪祥weixin_40376578weixin_48071367Jiang_Immortalsweixin_45392838和光-ycl长沙何书桓m0_59123733古月氵阳光一直都在iohgy2301_76764655oooqoyooolspppppp褪色的记忆探索者_SHUjackey0135th1157828416qq_46311760真命天子xy。。。770yupo555chun_yinLeoidea-seconddsqykfMessi20145102weixin_44953803AriSan_XDYWSFFJweixin_38912254thepolestweixin_46378822?Endless??王早起Morningsweixin_40328981kerwin666666穆如清风-weixin_45676656叫牛奶的特仑苏2301_780365932301_79547616ll489z726218yHUCONcongSakura20201017Idie_m0_69349550RSDLerAntisocialm405230843鱼-qqhamzsyI123456Tweixin_45526931像风像雾又像雨6qq_36497656憨化龙猫白小岩TobetterManBrainsa16225447_aasdfweixin_48030023m0_66285644129beerpython_dizhongdi2301_79442945梦想变成光浩浩是我阿a2233883飞凡850luojing913weixin_39750622偏爱中国红DemonHunter211paperxxxzuojunming0600wink537u010544479小赵啊。嗯哼weixin_44218547可乐小超人211统计黎罹巨量HTTP一只快乐的小龟m0_70472418高祺m0_740603742301_79349731qq_39169688刘635wlwlwl0511野心未泯weixin_48511181poptarCape0915qq_16068833lycdc_zbyhehe懒-洋洋ghjud语音识别_小白桦曳qq_36722565211统计课堂充电君h15728721597qq_35154867鲸云夕陌lobster.胖鸽qq_51644107极客曰报oceanaom0_698761492301_79201558我就笑吧728醴碘大金MrkingCN930804日向小太阳weixin_54669750weixin_40701906iso2302301_76718493satellite7m0_68703686m0_63268594随风56qq_34029746宝贝今晚想吃啥slc18698726058咚咚锵的小波波zz_comeonwfshan06弈诚毛毛mbwlhf666ai__xiaoyangskyvoice2二七学长Strawberrerweixin_54896532华为OD面试指南叶庭云xydlovecjjqq_40412114Yuyang7372灰格wude111weixin_45428949Yanzhenjinghan_Charlie_1541satday725fuwy2301_76386433Curtain?(o^^o)?weixin_47520923qq_57626668m0_55989410a1780335808zzc2323Pandaserxyvybun颜肆陆山月记469c_s_____d_na1205683快乐晓笑98li53957741小武落泪zou_jian_刘建杰麻烦指证批评weixin_41756759叶尖上跳跃zrr12620everylovebc20250401qq_41411881m0_73519261陆臣杰朽木莲尘埃里的月光chenfulong005m0_37680878笨小孩Quxyz9527xyzkidmixweixin_44829335夏701zkk9390wudehua55555陈嘉嘉嘉嘉m0_38065375m0_46178874LYB_383409060YANYAOHUI_weixin_46067110l66624766编程大狮Katherine_192201_75836058m0_61916878Nanxious_LeotxfTrapsinat_23581595求学80后魔力矩阵nana_lhan码码小猿m0_49149738小头先森starrym0_49217555m0_64335638啥时开始weixin_69926086黄开炎Csdsdfarelaxing520aaaaaaaaddd2weixin_57061799pighead_1016奶茶舒胡蕾qq_43630189m0_69898704babykikiim0_596739112301_78564831m0_57230756qq_20754835 我是我儿爹qq_36877641m0_51528231weixin_53774310依然饭特稀_0574麦唛weixin_57126192weixin_44542985LZ创小白abnerm0_69859729春待月九qq_457601422301_77334809宇宙小菜菜QiranMuqq_52184092weixin_68946462帅帅的单身狗風火雷btbtmy?CYX441sovelenPhilowen2301_77965635NZGMTCAHm0_71178254l501900JohnWong007m0_70902221cldyg222qq_57897148m0_63740483evil2333WayneBinqq_46663976weixin_46190290chaosDY不争之德shuxuejianmo314白白。猫qq_47348533一枚程序员Benjamin_Youngfadedol棒棒哒小纸盒zIT_baibanweixin_382700190Ooweixin_586164122201_75706509gaokeaiya三窗Gatorade001m0_69453712weixin_46808723MOB2333qq_42114034welkin_lgm0_58448384Energia_Buranaxl668weixin_44855103qq_30681957我随心动水日之秋李无趣qq_37510379qq_38057104HsatrY~404图灵软件技术十螺纹AllenSun34这个年纪你怎么睡得着的大约在雨季。大羊腿2201_76066153qq_27794783聂鲁达不哒ncuthmknvoianSD1wayTyphoonnlttzy2301_77693996jiagang_布丁_码到成功别任性m0_70818773let_wangu010181864gt17969hshsjptmachinaa48631685zzerhtdengine_men2301_77601254qq_45871585二分法QwQweixin_48950343Arlene_08Diamond我寄给你星河AlexZ.Robotcomicm0_59457021qq_57218083dawn_breeze_sleepyshamanGLewis妳っ杲昃Transient_520angledou杳冥、伈刼qq_34052077Anoxia_Tianm0_67716204relaxO氧awnusk0o圈圈o0爱jin946再不学就来不及了lwj070117嗨和嗨lijiejie1KYRIE00走心371tcyxy123THAST小猫了xy66_2301_76672683咖啡凉了_hy摆摆渡人wL0NG四幺7thheavenWML狄水云笙打铁匠Mr_Lee等你下课ソCynthiacircle天海一直在AIWbigd⊙▽⊙xqq_42796828NowLoadYweixin_44170459weixin_40352994apinapplePython_15TheF0oreVeR晚椒qq_46010292vvv贝塔西塔simonlijilirabbit3691weixin_45202744qq779670222nbmnqa_ws_99A1234587_000qq_38343917话唠画废ZtLoveWym五分QY199826dyz13hanhanyld2301_76678729MountVoomagneskanm0_74424405weixin_49078858假面游魂ChA0s007a646711278Phantom_ruinweixin_45749271qq_53270509Delanncym0_50111564扫地僧的扫帚stone_312m0_59039269慕容飞yishujiayouD.Woodwardfor?;;?qq_36042424小张tostringHenderson_XSince_julyweixin_49142770sinat_39136647weixin_67033104superliuchun夏目幸一slidevapnik222WwwhaiXuuDarry00HuMHkmr9725qq_576215582301_76608612DeemeDDBean_3qq_42939186m0_54165145m0_63405099秃头猿是可可粉Layeyeye37咕噜Kerwinmyccver酷kyHoly姜姜weixin_46800063彭彭丫qq_44488613m0_69353409X_Jun005千君殇普通网友zz_qq32107ZacharyZzlm0_74935297优秀木木.归居于尘°Floraaaa?无敌番茄酱Mr.利威尔多尝试多记录多积累希望_睿智weixin_51440866一不小心嗑到牙了Fyzqae炫荡之灵➹枫丹白露喝摩卡的狐狸小糖宝哈佛215柏劲松
