# in easynlp/appzoo/text2image_generation/predictor.pydef preprocess(self, in_data): if not in_data: raise RuntimeError("Input data should not be None.")if not isinstance(in_data, list): in_data = https://www.it610.com/article/[in_data] rst = {"idx": [], "input_ids": []} max_seq_length = -1 for record in in_data: if "sequence_length" not in record: break max_seq_length = max(max_seq_length, record["sequence_length"]) max_seq_length = self.sequence_length if (max_seq_length == -1) else max_seq_lengthfor record in in_data: text= record[self.first_sequence] try: self.MUTEX.acquire() text_ids = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(text)) text_ids = text_ids[: self.text_len] n_pad = self.text_len - len(text_ids) text_ids += [self.pad_id] * n_pad text_ids = np.array(text_ids) + self.img_vocab_sizefinally: self.MUTEX.release()rst["idx"].append(record["idx"]) rst["input_ids"].append(text_ids) return rst

# in easynlp/appzoo/text2image_generation/model.pydef generate(self, inputs, top_k=100, temperature=1.0): cidx = inputs sample = True steps = 256 for k in range(steps): x_cond = cidx logits, _ = self.transformer(x_cond) # pluck the logits at the final step and scale by temperature logits = logits[:, -1, :] / temperature # optionally crop probabilities to only the top k options if top_k is not None: logits = self.top_k_logits(logits, top_k) # apply softmax to convert to probabilities probs = torch.nn.functional.softmax(logits, dim=-1) # sample from the distribution or take the most likely if sample: ix = torch.multinomial(probs, num_samples=1) else: _, ix = torch.topk(probs, k=1, dim=-1) # append to the sequence and continue cidx = torch.cat((cidx, ix), dim=1) img_idx = cidx[:, 32:] return img_idx

import base64 from io import BytesIO from PIL import Imageimg = Image.open(fn) img_buffer = BytesIO() img.save(img_buffer, format=img.format) byte_data = https://www.it610.com/article/img_buffer.getvalue() base64_str = base64.b64encode(byte_data) # bytes

# train https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/painter_text2image/MUGE_train_text_imgbase64.tsv# valid https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/painter_text2image/MUGE_val_text_imgbase64.tsv# test https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/painter_text2image/MUGE_test.text.tsv

easynlp \ --mode=train \ --worker_gpu=1 \ --tables=MUGE_val_text_imgbase64.tsv,MUGE_val_text_imgbase64.tsv \ --input_schema=idx:str:1,text:str:1,imgbase64:str:1 \ --first_sequence=text \ --second_sequence=imgbase64 \ --checkpoint_dir=./finetuned_model/ \ --learning_rate=4e-5 \ --epoch_num=1 \ --random_seed=42 \ --logging_steps=100 \ --save_checkpoint_steps=1000 \ --sequence_length=288 \ --micro_batch_size=16 \ --app_name=text2image_generation \ --user_defined_parameters=' pretrain_model_name_or_path=alibaba-pai/pai-painter-large-zh size=256 text_len=32 img_len=256 img_vocab_size=16384 '

easynlp \ --mode=predict \ --worker_gpu=1 \ --tables=MUGE_test.text.tsv \ --input_schema=idx:str:1,text:str:1 \ --first_sequence=text \ --outputs=./T2I_outputs.tsv \ --output_schema=idx,text,gen_imgbase64 \ --checkpoint_dir=./finetuned_model/ \ --sequence_length=288 \ --micro_batch_size=8 \ --app_name=text2image_generation \ --user_defined_parameters=' size=256 text_len=32 img_len=256 img_vocab_size=16384 '

# 直接构建pipeline default_ecommercial_pipeline = pipeline("pai-painter-commercial-base-zh")# 模型预测 data = https://www.it610.com/article/["宽松T恤"] results = default_ecommercial_pipeline(data)# results的每一条是生成图像的base64编码# base64转换为图像 def base64_to_image(imgbase64_str): image = Image.open(BytesIO(base64.urlsafe_b64decode(imgbase64_str))) return image# 保存以文本命名的图像 for text, result in zip(data, results): imgpath = '{}.png'.format(text) imgbase64_str = result['gen_imgbase64'] image = base64_to_image(imgbase64_str) image.save(imgpath) print('text: {}, save generated image: {}'.format(text, imgpath))

• 自然风光场景：“pai-painter-scenery-base-zh”
• 中国山水画场景：“pai-painter-painting-base-zh”
  在上面的代码当中替换“pai-painter-commercial-base-zh”，就可以直接体验，欢迎试用。对于用户Fine-tune的文图生成模型，我们也开放了自定义模型加载的Pipeline接口：

# 加载模型，构建pipeline local_model_path = ... text_to_image_pipeline = pipeline("text2image_generation", local_model_path)# 模型预测 data = https://www.it610.com/article/["xxxx"] results = text_to_image_pipeline(data)# results的每一条是生成图像的base64编码

1. Chengyu Wang, Minghui Qiu, Taolin Zhang, Tingting Liu, Lei Li, Jianing Wang, Ming Wang, Jun Huang, Wei Lin. EasyNLP: A Comprehensive and Easy-to-use Toolkit for Natural Language Processing. arXiv
2. Aditya Ramesh, Mikhail Pavlov, Gabriel Goh, Scott Gray, Chelsea Voss, Alec Radford, Mark Chen, Ilya Sutskever. Zero-Shot Text-to-Image Generation. ICML 2021: 8821-8831
3. Ming Ding, Zhuoyi Yang, Wenyi Hong, Wendi Zheng, Chang Zhou, Da Yin, Junyang Lin, Xu Zou, Zhou Shao, Hongxia Yang, Jie Tang. CogView: Mastering Text-to-Image Generation via Transformers. NeurIPS 2021: 19822-19835
4. Han Zhang, Weichong Yin, Yewei Fang, Lanxin Li, Boqiang Duan, Zhihua Wu, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. ERNIE-ViLG: Unified Generative Pre-training for Bidirectional Vision-Language Generation. arXiv
5. Peng Wang, An Yang, Rui Men, Junyang Lin, Shuai Bai, Zhikang Li, Jianxin Ma, Chang Zhou, Jingren Zhou, Hongxia Yang. Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework. ICML 2022
6. Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, Mark Chen. Hierarchical Text-Conditional Image Generation with CLIP Latents. arXiv
7. Van Den Oord A, Vinyals O. Neural discrete representation learning. NIPS 2017
8. Esser P, Rombach R, Ommer B. Taming transformers for high-resolution image synthesis. CVPR 2021: 12873-12883.
9. Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kamyar Seyed Ghasemipour, Burcu Karagol Ayan, S. Sara Mahdavi, Rapha Gontijo Lopes, Tim Salimans, Jonathan Ho, David J. Fleet, Mohammad Norouzi: Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding. arXiv
10. Jiahui Yu, Yuanzhong Xu, Jing Yu Koh, Thang Luong, Gunjan Baid, Zirui Wang, Vijay Vasudevan, Alexander Ku, Yinfei Yang, Burcu Karagol Ayan, Ben Hutchinson, Wei Han, Zarana Parekh, Xin Li, Han Zhang, Jason Baldridge, Yonghui Wu. Scaling Autoregressive Models for Content-Rich Text-to-Image Generation. arXiv

推荐阅读

• 

  高三复读生该如何减压？身心健康才是成功教育！
• 

  ps褶皱文字怎么做 photoshop制作褶皱文字效果的操作步骤
• 

  昆仑山在哪 昆仑山的位置
• 

  日异月殊的意思 日异月殊的出处
• 

  高效能慢生活践行第626天1103-2020
• 

  等额本息和等额本金哪个划算 等额本息和等额本金哪个划算30年
• 

  浴盐多久用一次好
• 

  板鞋五个洞系鞋带的方法 板鞋五个洞系鞋带的方法图解
• 

  血见愁的功效与作用 血见愁
• 

  烫头发的药水
• 

  适合旅游的时候用的相机
• 

  天魔劫 天劫
• 

  盈滨海岸 一米海岸
• 

  用完脱毛仪之后可以用身体乳么
• 

  苦苣菜的做法和功效
• 

  电视坏了可以改成什么样的，电视坏了可以改成什么样的电视
• 

  虎牙直播一直正在加载，虎牙直播一直正在加载怎么回事
• 

  建安文学对后代文学有何积极影响
• 

  2022年5月希腊移民入境人数激增
• 

  想回农村养兔子最少得养多少？