建设网站必须用dns,自己做本地视频网站,家庭装潢设计,做企业网站需要资质吗浦语灵笔的图文理解及创作部署
浦语灵笔是基于书生浦语大语言模型研发的视觉-语言大模型#xff0c;提供出色的图文理解和创作能力#xff0c;结合了视觉和语言的先进技术#xff0c;能够实现图像到文本、文本到图像的双向转换。使用浦语灵笔大模型可以轻松的创作一篇图文推…浦语·灵笔的图文理解及创作部署
浦语·灵笔是基于书生·浦语大语言模型研发的视觉-语言大模型提供出色的图文理解和创作能力结合了视觉和语言的先进技术能够实现图像到文本、文本到图像的双向转换。使用浦语·灵笔大模型可以轻松的创作一篇图文推文也能够轻松识别一张图片中的物体并生成对应的文本描述。
使用 InternStudio 中的 A100(1/4) * 2 机器和 internlm-xcomposer-7b 模型部署一个图文理解创作 Demo
克隆环境
/root/share/install_conda_env_internlm_base.sh xcomposer-demo激活环境
conda activate xcomposer-demo安装依赖
pip install transformers4.33.1 timm0.4.12 sentencepiece0.1.99 gradio3.44.4 markdown22.4.10 xlsxwriter3.1.2 einops accelerate模型下载
mkdir -p /root/model/Shanghai_AI_Laboratory
cp -r /root/share/temp/model_repos/internlm-xcomposer-7b /root/model/Shanghai_AI_Laboratory代码准备
cd /root/code
git clone https://gitee.com/internlm/InternLM-XComposer.git
cd /root/code/InternLM-XComposer
git checkout 3e8c79051a1356b9c388a6447867355c0634932ddemo运行
cd /root/code/InternLM-XComposer
python examples/web_demo.py \--folder /root/model/Shanghai_AI_Laboratory/internlm-xcomposer-7b \--num_gpus 1 \--port 6006运行效果
完成 Lagent 工具调用 Demo 创作部署
环境准备.
python -m pip install --upgrade pippip install modelscope1.9.5
pip install transformers4.35.2
pip install streamlit1.24.0
pip install sentencepiece0.1.99
pip install accelerate0.24.1模型下载
mkdir -p /root/model/Shanghai_AI_Laboratory
cp -r /root/share/temp/model_repos/internlm-chat-7b /root/model/Shanghai_AI_LaboratoryLagent 安装
cd /root/code
git clone https://gitee.com/internlm/lagent.git
cd /root/code/lagent
git checkout 511b03889010c4811b1701abb153e02b8e94fb5e # 尽量保证和教程commit版本一致
pip install -e . # 源码安装修改代码 由于代码修改的地方比较多大家直接将 /root/code/lagent/examples/react_web_demo.py 内容替换为以下代码
import copy
import osimport streamlit as st
from streamlit.logger import get_loggerfrom lagent.actions import ActionExecutor, GoogleSearch, PythonInterpreter
from lagent.agents.react import ReAct
from lagent.llms import GPTAPI
from lagent.llms.huggingface import HFTransformerCasualLMclass SessionState:def init_state(self):Initialize session state variables.st.session_state[assistant] []st.session_state[user] []#action_list [PythonInterpreter(), GoogleSearch()]action_list [PythonInterpreter()]st.session_state[plugin_map] {action.name: actionfor action in action_list}st.session_state[model_map] {}st.session_state[model_selected] Nonest.session_state[plugin_actions] set()def clear_state(self):Clear the existing session state.st.session_state[assistant] []st.session_state[user] []st.session_state[model_selected] Noneif chatbot in st.session_state:st.session_state[chatbot]._session_history []class StreamlitUI:def __init__(self, session_state: SessionState):self.init_streamlit()self.session_state session_statedef init_streamlit(self):Initialize Streamlits UI settings.st.set_page_config(layoutwide,page_titlelagent-web,page_icon./docs/imgs/lagent_icon.png)# st.header(:robot_face: :blue[Lagent] Web Demo , dividerrainbow)st.sidebar.title(模型控制)def setup_sidebar(self):Setup the sidebar for model and plugin selection.model_name st.sidebar.selectbox(模型选择, options[gpt-3.5-turbo,internlm])if model_name ! st.session_state[model_selected]:model self.init_model(model_name)self.session_state.clear_state()st.session_state[model_selected] model_nameif chatbot in st.session_state:del st.session_state[chatbot]else:model st.session_state[model_map][model_name]plugin_name st.sidebar.multiselect(插件选择,optionslist(st.session_state[plugin_map].keys()),default[list(st.session_state[plugin_map].keys())[0]],)plugin_action [st.session_state[plugin_map][name] for name in plugin_name]if chatbot in st.session_state:st.session_state[chatbot]._action_executor ActionExecutor(actionsplugin_action)if st.sidebar.button(清空对话, keyclear):self.session_state.clear_state()uploaded_file st.sidebar.file_uploader(上传文件, type[png, jpg, jpeg, mp4, mp3, wav])return model_name, model, plugin_action, uploaded_filedef init_model(self, option):Initialize the model based on the selected option.if option not in st.session_state[model_map]:if option.startswith(gpt):st.session_state[model_map][option] GPTAPI(model_typeoption)else:st.session_state[model_map][option] HFTransformerCasualLM(/root/model/Shanghai_AI_Laboratory/internlm-chat-7b)return st.session_state[model_map][option]def initialize_chatbot(self, model, plugin_action):Initialize the chatbot with the given model and plugin actions.return ReAct(llmmodel, action_executorActionExecutor(actionsplugin_action))def render_user(self, prompt: str):with st.chat_message(user):st.markdown(prompt)def render_assistant(self, agent_return):with st.chat_message(assistant):for action in agent_return.actions:if (action):self.render_action(action)st.markdown(agent_return.response)def render_action(self, action):with st.expander(action.type, expandedTrue):st.markdown(p styletext-align: left;display:flex; span stylefont-size:14px;font-weight:600;width:70px;text-align-last: justify;插 件/spanspan stylewidth:14px;text-align:left;display:block;:/spanspan styleflex:1; # noqa E501 action.type /span/p,unsafe_allow_htmlTrue)st.markdown(p styletext-align: left;display:flex; span stylefont-size:14px;font-weight:600;width:70px;text-align-last: justify;思考步骤/spanspan stylewidth:14px;text-align:left;display:block;:/spanspan styleflex:1; # noqa E501 action.thought /span/p,unsafe_allow_htmlTrue)if (isinstance(action.args, dict) and text in action.args):st.markdown(p styletext-align: left;display:flex;span stylefont-size:14px;font-weight:600;width:70px;text-align-last: justify; 执行内容/spanspan stylewidth:14px;text-align:left;display:block;:/span/p, # noqa E501unsafe_allow_htmlTrue)st.markdown(action.args[text])self.render_action_results(action)def render_action_results(self, action):Render the results of action, including text, images, videos, andaudios.if (isinstance(action.result, dict)):st.markdown(p styletext-align: left;display:flex;span stylefont-size:14px;font-weight:600;width:70px;text-align-last: justify; 执行结果/spanspan stylewidth:14px;text-align:left;display:block;:/span/p, # noqa E501unsafe_allow_htmlTrue)if text in action.result:st.markdown(p styletext-align: left; action.result[text] /p,unsafe_allow_htmlTrue)if image in action.result:image_path action.result[image]image_data open(image_path, rb).read()st.image(image_data, captionGenerated Image)if video in action.result:video_data action.result[video]video_data open(video_data, rb).read()st.video(video_data)if audio in action.result:audio_data action.result[audio]audio_data open(audio_data, rb).read()st.audio(audio_data)def main():logger get_logger(__name__)# Initialize Streamlit UI and setup sidebarif ui not in st.session_state:session_state SessionState()session_state.init_state()st.session_state[ui] StreamlitUI(session_state)else:st.set_page_config(layoutwide,page_titlelagent-web,page_icon./docs/imgs/lagent_icon.png)# st.header(:robot_face: :blue[Lagent] Web Demo , dividerrainbow)model_name, model, plugin_action, uploaded_file st.session_state[ui].setup_sidebar()# Initialize chatbot if it is not already initialized# or if the model has changedif chatbot not in st.session_state or model ! st.session_state[chatbot]._llm:st.session_state[chatbot] st.session_state[ui].initialize_chatbot(model, plugin_action)for prompt, agent_return in zip(st.session_state[user],st.session_state[assistant]):st.session_state[ui].render_user(prompt)st.session_state[ui].render_assistant(agent_return)# User input form at the bottom (this part will be at the bottom)# with st.form(keymy_form, clear_on_submitTrue):if user_input : st.chat_input():st.session_state[ui].render_user(user_input)st.session_state[user].append(user_input)# Add file uploader to sidebarif uploaded_file:file_bytes uploaded_file.read()file_type uploaded_file.typeif image in file_type:st.image(file_bytes, captionUploaded Image)elif video in file_type:st.video(file_bytes, captionUploaded Video)elif audio in file_type:st.audio(file_bytes, captionUploaded Audio)# Save the file to a temporary location and get the pathfile_path os.path.join(root_dir, uploaded_file.name)with open(file_path, wb) as tmpfile:tmpfile.write(file_bytes)st.write(fFile saved at: {file_path})user_input 我上传了一个图像路径为: {file_path}. {user_input}.format(file_pathfile_path, user_inputuser_input)agent_return st.session_state[chatbot].chat(user_input)st.session_state[assistant].append(copy.deepcopy(agent_return))logger.info(agent_return.inner_steps)st.session_state[ui].render_assistant(agent_return)if __name__ __main__:root_dir os.path.dirname(os.path.dirname(os.path.abspath(__file__)))root_dir os.path.join(root_dir, tmp_dir)os.makedirs(root_dir, exist_okTrue)main()Demo运行
streamlit run /root/code/lagent/examples/react_web_demo.py --server.address 127.0.0.1 --server.port 6006运行效果
