# in easynlp/appzoo/text2image_generation/predictor.pydef preprocess(self, in_data): if not in_data: raise RuntimeError("Input data should not be None.")if not isinstance(in_data, list): in_data = https://www.it610.com/article/[in_data] rst = {"idx": [], "input_ids": []} max_seq_length = -1 for record in in_data: if "sequence_length" not in record: break max_seq_length = max(max_seq_length, record["sequence_length"]) max_seq_length = self.sequence_length if (max_seq_length == -1) else max_seq_lengthfor record in in_data: text= record[self.first_sequence] try: self.MUTEX.acquire() text_ids = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(text)) text_ids = text_ids[: self.text_len] n_pad = self.text_len - len(text_ids) text_ids += [self.pad_id] * n_pad text_ids = np.array(text_ids) + self.img_vocab_sizefinally: self.MUTEX.release()rst["idx"].append(record["idx"]) rst["input_ids"].append(text_ids) return rst

# in easynlp/appzoo/text2image_generation/model.pydef generate(self, inputs, top_k=100, temperature=1.0): cidx = inputs sample = True steps = 256 for k in range(steps): x_cond = cidx logits, _ = self.transformer(x_cond) # pluck the logits at the final step and scale by temperature logits = logits[:, -1, :] / temperature # optionally crop probabilities to only the top k options if top_k is not None: logits = self.top_k_logits(logits, top_k) # apply softmax to convert to probabilities probs = torch.nn.functional.softmax(logits, dim=-1) # sample from the distribution or take the most likely if sample: ix = torch.multinomial(probs, num_samples=1) else: _, ix = torch.topk(probs, k=1, dim=-1) # append to the sequence and continue cidx = torch.cat((cidx, ix), dim=1) img_idx = cidx[:, 32:] return img_idx

import base64 from io import BytesIO from PIL import Imageimg = Image.open(fn) img_buffer = BytesIO() img.save(img_buffer, format=img.format) byte_data = https://www.it610.com/article/img_buffer.getvalue() base64_str = base64.b64encode(byte_data) # bytes

# train https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/painter_text2image/MUGE_train_text_imgbase64.tsv# valid https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/painter_text2image/MUGE_val_text_imgbase64.tsv# test https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/painter_text2image/MUGE_test.text.tsv

easynlp \ --mode=train \ --worker_gpu=1 \ --tables=MUGE_val_text_imgbase64.tsv,MUGE_val_text_imgbase64.tsv \ --input_schema=idx:str:1,text:str:1,imgbase64:str:1 \ --first_sequence=text \ --second_sequence=imgbase64 \ --checkpoint_dir=./finetuned_model/ \ --learning_rate=4e-5 \ --epoch_num=1 \ --random_seed=42 \ --logging_steps=100 \ --save_checkpoint_steps=1000 \ --sequence_length=288 \ --micro_batch_size=16 \ --app_name=text2image_generation \ --user_defined_parameters=' pretrain_model_name_or_path=alibaba-pai/pai-painter-large-zh size=256 text_len=32 img_len=256 img_vocab_size=16384 '

easynlp \ --mode=predict \ --worker_gpu=1 \ --tables=MUGE_test.text.tsv \ --input_schema=idx:str:1,text:str:1 \ --first_sequence=text \ --outputs=./T2I_outputs.tsv \ --output_schema=idx,text,gen_imgbase64 \ --checkpoint_dir=./finetuned_model/ \ --sequence_length=288 \ --micro_batch_size=8 \ --app_name=text2image_generation \ --user_defined_parameters=' size=256 text_len=32 img_len=256 img_vocab_size=16384 '

# 直接构建pipeline default_ecommercial_pipeline = pipeline("pai-painter-commercial-base-zh")# 模型预测 data = https://www.it610.com/article/["宽松T恤"] results = default_ecommercial_pipeline(data)# results的每一条是生成图像的base64编码# base64转换为图像 def base64_to_image(imgbase64_str): image = Image.open(BytesIO(base64.urlsafe_b64decode(imgbase64_str))) return image# 保存以文本命名的图像 for text, result in zip(data, results): imgpath = '{}.png'.format(text) imgbase64_str = result['gen_imgbase64'] image = base64_to_image(imgbase64_str) image.save(imgpath) print('text: {}, save generated image: {}'.format(text, imgpath))

• 自然风光场景：“pai-painter-scenery-base-zh”
• 中国山水画场景：“pai-painter-painting-base-zh”
  在上面的代码当中替换“pai-painter-commercial-base-zh”，就可以直接体验，欢迎试用。对于用户Fine-tune的文图生成模型，我们也开放了自定义模型加载的Pipeline接口：

# 加载模型，构建pipeline local_model_path = ... text_to_image_pipeline = pipeline("text2image_generation", local_model_path)# 模型预测 data = https://www.it610.com/article/["xxxx"] results = text_to_image_pipeline(data)# results的每一条是生成图像的base64编码

1. Chengyu Wang, Minghui Qiu, Taolin Zhang, Tingting Liu, Lei Li, Jianing Wang, Ming Wang, Jun Huang, Wei Lin. EasyNLP: A Comprehensive and Easy-to-use Toolkit for Natural Language Processing. arXiv
2. Aditya Ramesh, Mikhail Pavlov, Gabriel Goh, Scott Gray, Chelsea Voss, Alec Radford, Mark Chen, Ilya Sutskever. Zero-Shot Text-to-Image Generation. ICML 2021: 8821-8831
3. Ming Ding, Zhuoyi Yang, Wenyi Hong, Wendi Zheng, Chang Zhou, Da Yin, Junyang Lin, Xu Zou, Zhou Shao, Hongxia Yang, Jie Tang. CogView: Mastering Text-to-Image Generation via Transformers. NeurIPS 2021: 19822-19835
4. Han Zhang, Weichong Yin, Yewei Fang, Lanxin Li, Boqiang Duan, Zhihua Wu, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. ERNIE-ViLG: Unified Generative Pre-training for Bidirectional Vision-Language Generation. arXiv
5. Peng Wang, An Yang, Rui Men, Junyang Lin, Shuai Bai, Zhikang Li, Jianxin Ma, Chang Zhou, Jingren Zhou, Hongxia Yang. Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework. ICML 2022
6. Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, Mark Chen. Hierarchical Text-Conditional Image Generation with CLIP Latents. arXiv
7. Van Den Oord A, Vinyals O. Neural discrete representation learning. NIPS 2017
8. Esser P, Rombach R, Ommer B. Taming transformers for high-resolution image synthesis. CVPR 2021: 12873-12883.
9. Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kamyar Seyed Ghasemipour, Burcu Karagol Ayan, S. Sara Mahdavi, Rapha Gontijo Lopes, Tim Salimans, Jonathan Ho, David J. Fleet, Mohammad Norouzi: Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding. arXiv
10. Jiahui Yu, Yuanzhong Xu, Jing Yu Koh, Thang Luong, Gunjan Baid, Zirui Wang, Vijay Vasudevan, Alexander Ku, Yinfei Yang, Burcu Karagol Ayan, Ben Hutchinson, Wei Han, Zarana Parekh, Xin Li, Han Zhang, Jason Baldridge, Yonghui Wu. Scaling Autoregressive Models for Content-Rich Text-to-Image Generation. arXiv

推荐阅读

• 

  什么鸟儿是飞行家
• 

  古风笔名
• 

  怀孕初期吃什么好 怀孕初期饮食原则
• 

  redis查看当前是哪个数据库 查看redis数据库版本
• 

  【创融同学会征4动态】征4创融跑团戈壁挑战赛备战会议圆满成功
• 

  现在的英雄联盟满攻速是点那个符文？
• 

  蟹蒸出来怎么蟹黄变黑了
• 

  柠檬女孩什么意思 柠檬女孩的意思
• 

  tr4
• 

  佳能万能版清零软件-打印机清零软件怎么下载
• 

  Android开源日志库Logger
• 

  牵牛花种子什么时候种植 牵牛花种子什么时候种
• 

  西门子燃气灶显示灯解决方法有哪些？
• 

  夏季吃红辣椒可以预防中暑
• 

  大豆纤维被孔数越多越好吗
• 

  牛抄藤
• 

  usbcan分析仪
• 

  青海信创服务器在知乎上的评价如何？ 青海信创服务器怎么样知乎
• 

  怎样培殖蚯蚓 水蚯蚓培育方法
• 

  中国联通手机营业厅100M全国流量免费领 升级即可领取