text:
滑动异常

我是在BottomSheetDialogFragment中使用此控件，使用的时候发现一个Bug,拦截了我的触摸事件，点击事件是正常的，月视图状态我无法向下滑动隐藏BottomSheetDialogFragment
在我需要向下滑动隐藏BottomSheetDialogFragment的时候不能滑动，如果不选择年份滑动是正常的，选择19年的月份后月视图无法向下滑动，选择其他年份后月视图就能向下滑动

贴一下代码
xml文件
<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:orientation="vertical">
        <RelativeLayout
            android:id="@+id/layout"
            android:background="@color/white_color"
            android:layout_width="match_parent"
            android:layout_height="wrap_content">
                <TextView
                    android:text="@string/select_calendar"
                    android:textSize="@dimen/s18"
                    android:textStyle="bold"
                    android:textColor="@color/black_color"
                    android:gravity="center"
                    android:layout_centerInParent="true"
                    android:layout_width="wrap_content"
                    android:layout_height="@dimen/dp_40" />
                <TextView
                    android:id="@+id/tv_date"
                    android:text="@string/date_value"
                    android:textColor="@color/black_color"
                    android:gravity="center"
                    android:layout_alignParentEnd="true"
                    android:layout_marginEnd="@dimen/d20"
                    android:layout_width="wrap_content"
                    android:layout_height="wrap_content"
                    android:minHeight="@dimen/dp_40"/>
        </RelativeLayout>

        <com.haibin.calendarview.CalendarView
            android:id="@+id/calendarView"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:layout_below="@id/layout"
            android:background="@color/white_color"
            app:calendar_padding="10dp"
            app:current_day_lunar_text_color="@color/blue_color"
            app:current_day_text_color="@color/blue_color"
            app:current_month_lunar_text_color="@color/black_color"
            app:current_month_text_color="@color/black_color"
            app:lunar_text_size="8sp"
            app:max_year="2050"
            app:max_year_day="12"
            app:max_year_month="31"
            app:min_select_range="1"
            app:min_year="2008"
            app:min_year_day="1"
            app:min_year_month="1"
            app:month_view="com.tenement.ui.calendar.CustomRangeMonthView"
            app:month_view_show_mode="mode_fix"
            app:other_month_lunar_text_color="@color/gray"
            app:other_month_text_color="@color/gray"
            app:scheme_lunar_text_color="@color/gray"
            app:scheme_text="假"
            app:scheme_text_color="@color/red_color"
            app:scheme_theme_color="@color/orange_color"
            app:select_mode="range_mode"
            app:selected_lunar_text_color="@color/black_color"
            app:selected_text_color="@color/white_color"
            app:selected_theme_color="@color/blue_color"
            app:week_background="@color/white_color"
            app:week_bar_height="46dp"
            app:week_line_background="@android:color/transparent"
            app:week_line_margin="16dp"
            app:week_start_with="mon"
            app:week_text_size="10sp"
            app:week_view="com.blackcard.logan.util.calendar.CustomRangeWeekView"
            app:year_view="com.tenement.ui.calendar.CustomYearView"
            app:year_view_background="@color/white_color"
            app:year_view_day_text_color="@color/black_color"
            app:year_view_day_text_size="8sp"
            app:year_view_month_text_color="@color/blue_color"
            app:year_view_month_text_size="20sp"
            app:year_view_scheme_color="@color/red_color"
            app:year_view_select_text_color="@color/white_color"
            app:year_view_month_height="@dimen/d16_Miss_fang"
            app:year_view_week_height="@dimen/d16_Miss_fang"
            app:year_view_week_text_color="@color/black_shallow_color"
            app:year_view_week_text_size="7sp" />
</RelativeLayout>

调用代码
private void initRootView() {
        rootView = LayoutInflater.from(mContext).inflate(R.layout.view_calendar, null);
        calendarView = rootView.findViewById(R.id.calendarView);
        tv_date = rootView.findViewById(R.id.tv_date);
        tv_date.setText(String.format(getString(R.string.date_value), year = calendarView.getCurYear(), calendarView.getCurMonth()));
        rootView.getViewTreeObserver().addOnGlobalLayoutListener(() -> mBehavior.setPeekHeight(rootView.getHeight()));
        calendarView.post(() -> calendar_height = calendarView.getHeight());
        tv_date.setOnClickListener(v -> {
            if (!calendarView.isYearSelectLayoutVisible()) {
                calendarView.getLayoutParams().height = calendar_height + 300;
                calendarView.showYearSelectLayout(year);
                calendarView.requestLayout();
            }
        });
        //年视图左右滑动监听
        calendarView.setOnYearChangeListener(year -> tv_date.setText((this.year = year) + "年"));

        //打开关闭年视图监听
        calendarView.setOnYearViewChangeListener(isClose -> {
            if (!isClose) tv_date.setText(year + "年");
            else calendarView.getLayoutParams().height = ViewGroup.LayoutParams.WRAP_CONTENT;
        });
        calendarView.setOnMonthChangeListener((year, month) -> {
            tv_date.setText(String.format(getString(R.string.date_value), this.year = year, month));
        });
        calendarView.setOnCalendarRangeSelectListener(new CalendarView.OnCalendarRangeSelectListener() {
            @Override
            public void onCalendarSelectOutOfRange(Calendar calendar) {
                //范围选择超出范围越界
            }

            @Override
            public void onSelectOutOfRange(Calendar calendar, boolean isOutOfMinRange) {
                //选择范围超出范围
            }

            @Override
            public void onCalendarRangeSelect(Calendar calendar, boolean isEnd) {
                if (isEnd) {
                    calendarView.postDelayed(() -> {
                        eDate = calendar.getTimeInMillis();
                        if (listener != null) listener.onFinish(sDate, eDate);
                        close(true);
                    }, 150);
                } else sDate = calendar.getTimeInMillis();
            }
        });
    }

CalendarView和周月视图本身没有做重写事件的操作的，所有事件拦截的操作在CalenarLayout里面，你检查一下有没有重写了什么onTouch、onTouchEvent之类的事件，如果有的话应该是事件被消耗了，事实上直接这样用应该是没问题的，拦截事件应该是日历上层的BottomSheetDialogFragment里面的根布局

@huanghaibin-dev 我没有重写onTouch、onTouchEvent，其他的view可以正常滑动
贴一下代码
/**
 * Created by Logan on 2019/6/13.
 */
@SuppressLint("ValidFragment")
public class SelectRangeCalendarDialog extends BottomSheetDialogFragment {
    private Context mContext;
    protected BottomSheetBehavior mBehavior;
    private BottomSheetDialog dialog;
    private View rootView;
    private CalendarView calendarView;
    private TextView tv_date;

    private OnRangeListener listener;
    private long sDate, eDate;
    private int calendar_height;
    private int year;

    public SelectRangeCalendarDialog(OnRangeListener listener) {
        this.listener = listener;
    }

    @Override
    public void onAttach(Context context) {
        super.onAttach(context);
        this.mContext = context;
    }

    @Override
    public void onStart() {
        super.onStart();
        Window window = getDialog().getWindow();
        if (window != null) {
            WindowManager.LayoutParams windowParams = window.getAttributes();
            //这里设置透明度
            if (windowParams != null) {
                windowParams.dimAmount = 0.5f;
                window.setAttributes(windowParams);
            }
        }
        mBehavior.setState(BottomSheetBehavior.STATE_EXPANDED);
    }

    @Override
    public void onDestroy() {
        //解除缓存 View 和当前 ViewGroup 的关联
        ((ViewGroup) (rootView.getParent())).removeView(rootView);
        mBehavior.setState(BottomSheetBehavior.STATE_COLLAPSED);
        super.onDestroy();
    }

    @NotNull
    @Override
    public Dialog onCreateDialog(Bundle savedInstanceState) {
        dialog = new BottomSheetDialog(mContext, getTheme());
        if (rootView == null) {
            //缓存下来的 View 当为空时才需要初始化 并缓存
            initRootView();
        }
        dialog.setContentView(rootView);
        View parentView = (View) rootView.getParent();
        CoordinatorLayout.LayoutParams params = (CoordinatorLayout.LayoutParams) parentView.getLayoutParams();
        params.gravity = Gravity.TOP | Gravity.CENTER_HORIZONTAL;
        parentView.setLayoutParams(params);
        mBehavior = BottomSheetBehavior.from(parentView);
        rootView.measure(0, 0);
        mBehavior.setHideable(true);
        ((View) rootView.getParent()).setBackgroundColor(Color.TRANSPARENT);
        if (mBehavior != null) mBehavior.setState(BottomSheetBehavior.STATE_EXPANDED);
        return dialog;
    }

    @SuppressLint("SetTextI18n")
    private void initRootView() {
        rootView = LayoutInflater.from(mContext).inflate(R.layout.view_calendar, null);
        calendarView = rootView.findViewById(R.id.calendarView);
        tv_date = rootView.findViewById(R.id.tv_date);
        tv_date.setText(String.format(getString(R.string.date_value), year = calendarView.getCurYear(), calendarView.getCurMonth()));
        rootView.getViewTreeObserver().addOnGlobalLayoutListener(() -> mBehavior.setPeekHeight(rootView.getHeight()));
        calendarView.post(() -> calendar_height = calendarView.getHeight());
        tv_date.setOnClickListener(v -> {
            if (!calendarView.isYearSelectLayoutVisible()) {
                calendarView.getLayoutParams().height = calendar_height + 300;
                calendarView.showYearSelectLayout(year);
                calendarView.requestLayout();
            }
        });
        //年视图左右滑动监听
        calendarView.setOnYearChangeListener(year -> tv_date.setText((this.year = year) + "年"));

        //打开关闭年视图监听
        calendarView.setOnYearViewChangeListener(isClose -> {
            if (!isClose) tv_date.setText(year + "年");
            else calendarView.getLayoutParams().height = ViewGroup.LayoutParams.WRAP_CONTENT;
        });
        calendarView.setOnMonthChangeListener((year, month) -> tv_date.setText(String.format(getString(R.string.date_value), this.year = year, month)));
        calendarView.setOnCalendarRangeSelectListener(new CalendarView.OnCalendarRangeSelectListener() {
            @Override
            public void onCalendarSelectOutOfRange(Calendar calendar) {
                //范围选择超出范围越界
            }

            @Override
            public void onSelectOutOfRange(Calendar calendar, boolean isOutOfMinRange) {
                //选择范围超出范围
            }

            @Override
            public void onCalendarRangeSelect(Calendar calendar, boolean isEnd) {
                if (isEnd) {
                    calendarView.postDelayed(() -> {
                        eDate = calendar.getTimeInMillis();
                        if (listener != null) listener.onFinish(sDate, eDate);
                        close(true);
                    }, 150);
                } else sDate = calendar.getTimeInMillis();
            }
        });
    }

    public SelectRangeCalendarDialog show(FragmentManager manager) {
        if (!this.isAdded()) {
//            show(manager, TAG);//有的地方调用会导致IllegalStateException: Can not perform this action after onSaveInstanceState
            //解决方法使用commitAllowingStateLoss提交
            FragmentTransaction ft = manager.beginTransaction();
            ft.add(this, getClass().getSimpleName());
            ft.commitAllowingStateLoss();//注意这里使用commitAllowingStateLoss()
        }
        return this;
    }

    public boolean isShowing() {
        return dialog != null && dialog.isShowing();
    }

    /**
     * 使用关闭弹框 是否使用动画可选
     * 使用动画 同时切换界面Aty会卡顿 建议直接关闭
     *
     * @param isAnimation 是否使用动画效果
     */
    public void close(boolean isAnimation) {
        if (isAnimation) {
            if (mBehavior != null)
                mBehavior.setState(BottomSheetBehavior.STATE_HIDDEN);
        } else {
            dismiss();
        }
    }

    public interface OnRangeListener {
        void onFinish(long sDate, long eDate);
    }
}

authors: huanghaibin-dev loyilin

repo: huanghaibin-dev/CalendarView


text:
sanity-test fails despite installing optional [camera obs] dependencies

BUG REPORT
High Level Description
During make sanity-test,  without running the optional panda3D dependencies cause sanity-test to fail.
SMARTS version
0.4.18
Previous associated issues
N/A
Steps to reproduce the bug
Follow the setup steps in README.md and run make sanity-test
Resulting and expected behaviour
test_smarts_doesnt_leak_tasks_after_reset test will sometimes throw a RendererException when all tests should pass.
Error logs and screenshots

======================================================================================================================================= FAILURES =======================================================================================================================================______________________________________________________________________________________________________________________ test_smarts_doesnt_leak_tasks_after_reset _______________________________________________________________________________________________________________________[gw4] linux -- Python 3.8.10 /home/sqy201x/SMARTS/.venv/bin/python3.8
smarts = <smarts.core.smarts.SMARTS object at 0x7f979c599610>
scenarios = <itertools.cycle object at 0x7f979ada7340>

    def test_smarts_doesnt_leak_tasks_after_reset(smarts, scenarios):
        """We have had issues in the past where we would forget to clean up tasks between episodes
        resulting in a gradual decay in performance, this test gives us a bit of a smoke screen
        against this class of regressions.

        See #237 for details
        """
        try:
            num_tasks_before_reset = len(
>               smarts.renderer._showbase_instance.taskMgr.mgr.getTasks()
            )
E           AttributeError: 'NoneType' object has no attribute '_showbase_instance'

smarts/core/tests/test_smarts.py:81: AttributeError

During handling of the above exception, another exception occurred:

smarts = <smarts.core.smarts.SMARTS object at 0x7f979c599610>
scenarios = <itertools.cycle object at 0x7f979ada7340>

    def test_smarts_doesnt_leak_tasks_after_reset(smarts, scenarios):
        """We have had issues in the past where we would forget to clean up tasks between episodes
        resulting in a gradual decay in performance, this test gives us a bit of a smoke screen
        against this class of regressions.

        See #237 for details
        """
        try:
            num_tasks_before_reset = len(
                smarts.renderer._showbase_instance.taskMgr.mgr.getTasks()
            )
        except Exception as e:
>           raise RendererException.required_to("test smarts_doesnt_leak_tasks_after_reset")
E           smarts.core.vehicle.RendererException: A renderer is required to test smarts_doesnt_leak_tasks_after_reset. You may not have installed the [camera-obs] dependencies required to render the camera sensor observations. Install them first using the command `pip install -e .[camera-obs]` at the source directory.

smarts/core/tests/test_smarts.py:84: RendererException
---------------------------------------------------------------------------------------------------------------------------------- Captured log call -----------------------------------------------------------------------------------------------------------------------------------WARNING  SMARTS:smarts.py:532 unable to create Renderer:  ModuleNotFoundError("No module named 'gltf'")
----------------------------------------------------------------------------------------------------------- generated xml file: /home/sqy201x/SMARTS/sanity_test_result.xml ------------------------------------------------------------------------------------------------------------=============================================================================================================================== short test summary info ================================================================================================================================FAILED smarts/core/tests/test_smarts.py::test_smarts_doesnt_leak_tasks_after_reset
============================================================================================================================= 1 failed, 7 passed in 44.76s =============================================================================================================================make: *** [Makefile:23: sanity-test] Error 1

System information

Operating system & Environment [Please note if using Docker]:
WSL2 Ubuntu 20.04 LTS for WIN11 21H2
Number of system cores available
6
Python version [Please note if using Jupyter]
3.8.10
Python packages [e.g. from pip freeze]

absl-py==1.0.0
aiohttp==3.8.1
aiohttp-cors==0.7.0
aioredis==2.0.0
aiosignal==1.2.0
ale-py==0.7.3
anyio==3.4.0
argon2-cffi==21.1.0
astunparse==1.6.3
async-timeout==4.0.1
atari-py==0.2.9
attrs==20.3.0
Automat==20.2.0
backcall==0.2.0
beautifulsoup4==4.10.0
bleach==4.1.0
blessings==1.7
cached-property==1.5.2
cachetools==4.2.4
casadi==3.5.5
certifi==2021.10.8
cffi==1.15.0
charset-normalizer==2.0.9
click==8.0.3
cloudpickle==1.3.0
colorama==0.4.4
colorful==0.5.4
commonmark==0.9.1
constantly==15.1.0
coverage==6.2
cycler==0.11.0
debugpy==1.5.1
decorator==5.1.0
defusedxml==0.7.1
dm-tree==0.1.6
entrypoints==0.3
execnet==1.9.0
filelock==3.4.0
fonttools==4.28.3
frozenlist==1.2.0
future==0.18.2
gast==0.3.3
gitdb==4.0.9
GitPython==3.1.24
google==3.0.0
google-api-core==2.2.2
google-auth==1.35.0
google-auth-oauthlib==0.4.6
google-pasta==0.2.0
googleapis-common-protos==1.53.0
gpustat==0.6.0
grpcio==1.37.0
gym==0.21.0
h5py==2.10.0
hyperlink==21.0.0
idna==3.3
ijson==3.1.4
imageio==2.13.1
importlib-metadata==4.8.2
importlib-resources==5.4.0
incremental==21.3.0
iniconfig==1.1.1
ipykernel==6.6.0
ipython==7.30.1
ipython-genutils==0.2.0
jedi==0.18.1
Jinja2==3.0.3
joblib==1.1.0
jsonpatch==1.32
jsonpointer==2.2
jsonschema==4.2.1
jupyter-client==7.1.0
jupyter-core==4.9.1
jupyter-server==1.12.1
jupyter-server-mathjax==0.2.3
Keras-Preprocessing==1.1.2
kiwisolver==1.3.2
lz4==3.1.10
Markdown==3.3.6
MarkupSafe==2.0.1
matplotlib==3.5.0
matplotlib-inline==0.1.3
maturin==0.12.4
mistune==0.8.4
msgpack==1.0.3
multidict==5.2.0
nbconvert==5.6.1
nbdime==3.1.1
nbformat==5.1.3
nest-asyncio==1.5.4
networkx==2.6.3
numpy==1.18.5
nvidia-ml-py3==7.352.0
oauthlib==3.1.1
open-agent==0.1.195
opencensus==0.8.0
opencensus-context==0.1.2
opencv-python-headless==4.2.0.34
opengen==0.6.4
opt-einsum==3.3.0
packaging==21.3
Panda3D==1.10.10
panda3d-gltf==0.13
panda3d-simplepbr==0.9
pandas==1.3.4
pandocfilters==1.5.0
parso==0.8.3
pexpect==4.8.0
pickleshare==0.7.5
Pillow==8.4.0
pkg_resources==0.0.0
pluggy==1.0.0
prometheus-client==0.12.0
prompt-toolkit==3.0.23
protobuf==3.19.1
psutil==5.8.0
ptyprocess==0.7.0
py==1.11.0
py-cpuinfo==8.0.0
py-spy==0.3.11
pyasn1==0.4.8
pyasn1-modules==0.2.8
pybullet==3.0.6
pycparser==2.21
Pygments==2.10.0
pyparsing==3.0.6
pyrsistent==0.18.0
pytest==6.2.5
pytest-benchmark==3.4.1
pytest-cov==3.0.0
pytest-forked==1.3.0
pytest-notebook==0.6.1
pytest-xdist==2.4.0
python-dateutil==2.8.2
python-xlib==0.31
python_bindings_open_agent_solver==0.0.9
pytz==2021.3
PyWavelets==1.2.0
PyYAML==6.0
pyzmq==22.3.0
ray==1.0.1.post1
redis==3.4.1
requests==2.26.0
requests-oauthlib==1.3.0
retry==0.9.2
rich==10.15.2
rsa==4.8
Rtree==0.9.7
scikit-image==0.19.0
scikit-learn==1.0.1
scipy==1.4.1
Send2Trash==1.8.0
setuptools-scm==6.3.2
sh==1.14.2
Shapely==1.8.0
six==1.16.0
sklearn==0.0
-e git+https://github.com/huawei-noah/SMARTS.git@3d69ba1f5b9995ee8b12638676d92e5f4750f3a5#egg=smarts
smmap==5.0.0
sniffio==1.2.0
soupsieve==2.3.1
tableprint==0.9.1
tabulate==0.8.9
tensorboard==2.2.2
tensorboard-plugin-wit==1.8.0
tensorboardX==2.4.1
tensorflow==2.2.1
tensorflow-estimator==2.2.0
termcolor==1.1.0
terminado==0.12.1
testpath==0.5.0
threadpoolctl==3.0.0
tifffile==2021.11.2
toml==0.10.2
tomli==1.2.2
torch==1.4.0
torchfile==0.1.0
torchvision==0.5.0
tornado==6.1
traitlets==5.1.1
trimesh==3.9.29
Twisted==21.7.0
typing_extensions==4.0.1
urllib3==1.26.7
visdom==0.1.8.9
wcwidth==0.2.5
webencodings==0.5.1
websocket-client==1.2.1
Werkzeug==2.0.2
wrapt==1.13.3
yarl==1.7.2
yattag==1.14.0
zipp==3.6.0
zope.interface==5.4.0

SUMO version
1.11.0
Date when bug occurred
06/12/2021
Impact [If known]

The base reason behind this failure even if you do install [camera-obs] is that the system rendering dependencies such as x11 may be missing.

source: gharchive/issue

authors: Gamenot qianyi-sun

repo: huawei-noah/SMARTS

license: MIT

license_type: permissive

license_source: github-api


text:
dataset

when i run your notebook, i get:
Number of examples in each set:
Training: 18545
Validation: 4094
Test: 9850
but you reported:
Number of examples in each set:
Training: 1020
Validation: 1087
Test: 836
please, can you explain me, why this happen?

Thanks for the heads up @saharahmadisohraviyeh! It looks like I initially ran the notebook with fewer recordings (probably 3) which would explain the difference that you noticed - rerunning it gives me the same numbers as you. I just updated the tutorial accordingly.

Hi,thanks for answer.good luck.
On Sunday, May 12, 2024 at 09:14:55 PM GMT+3:30, Hubert Banville @.***> wrote:
Thanks for the heads up @saharahmadisohraviyeh! It looks like I initially ran the notebook with fewer recordings (probably 3) which would explain the difference that you noticed - rerunning it gives me the same numbers as you. I just updated the tutorial accordingly.
—
Reply to this email directly, view it on GitHub, or unsubscribe.
You are receiving this because you were mentioned.Message ID: @.***>

authors: hubertjb saharahmadisohraviyeh

repo: hubertjb/dl-eeg-tutorial


text:
Configuration file "nexus.properties" not created during deployment

Nexus won't start in case of a missing nexus configuration, because there is no default configuration in place after deployment:
/srv/sonatype-work/nexus3/etc/nexus.properties
We should provide a default configuration with (at least) the following definitions:
## DO NOT EDIT - PUPPET MAINTAINS THIS FILE
##
# Jetty section
application-port=8081
application-host=0.0.0.0
nexus-args=${jetty.etc}/jetty.xml,${jetty.etc}/jetty-http.xml,${jetty.etc}/jetty-requestlog.xml

# Nexus section
nexus-edition=nexus-pro-edition
nexus-features=\
 nexus-pro-feature

+1

repo: hubspotdevops/puppet-nexus

license: mit

license_source: bigquery


text:
Add creation_code and creation_code_with_args methods to HuffConfig

Getting creation code allows for passing initCode as parameters to other smart contracts, such as to CREATE2 factories, constructors, or test functions.
New functions are indirectly tested by existing tests, but I can add some if necessary.
Would indirectly mitigate #24 as well.

I personally really like this, good thinking

source: gharchive/pull-request

authors: cheethas jameswenzel

repo: huff-language/foundry-huff


text:
Prevent URL from being printed for each API request

I can see no useful reason for printing the url other than debugging- otherwise all it does is clutter up output.

Thanks for the help. Now I have to figure out how to publish this change.... :)

authors: adamhooper tedivm

repo: huffpostdata/python-pollster


text:
🛑 Akkoma is down

In fed5db6, Akkoma (https://akkoma.chez.chat) was down:

HTTP code: 404
Response time: 309 ms

Resolved: Akkoma is back up in 88e3a5b after 6 minutes.

repo: hufman/upptime


text:
[Bug] hbase删除索引后无法插入数据

Bug Type (问题类型)
No response
Before submit

[X] 我已经确认现有的 Issues 与 FAQ 中没有相同 / 重复问题

Environment (环境信息)

Server Version: v0.12.x
Backend: hbase
OS: xx CPUs, xx G RAM, Centos 7.x
Data Size:  1.5亿 vertices, 1.3亿 edges

Expected & Actual behavior (期望与实际表现)
后端：Hbase
因查询需要，建立了几个全文索引和范围索引，如
schema.indexLabel("userByOrigins")
.onV(USER)
.by(ORIGINS)
.search()
.ifNotExist()
.create();
但发现，索引表占用存储过大

所以就执行了删除索引的操作
schema.indexLabel("userByOrigins").remove()
完成之后
1、索引表的大小并没有变化
2、Hbase无法再执行任何操作，包括查询和插入等，报错如下：

在hubble上执行插入顶点操作，报错如下：

请帮忙分析原因，感谢
但以上操作在同数据量的rocksdb上操作就没有问题（当然索引表的大小也是没有恢复，但可以正常查询和插入数据）
Vertex/Edge example (问题点 / 边数据举例)
No response
Schema [VertexLabel, EdgeLabel, IndexLabel] (元数据结构)
No response

索引表的大小并没有变化

索引表的大小并没有变化，LSM-Tree的删除操作不会立即删除数据，删除操作之后还未触发compaction，等compaction之后应该会清除磁盘数据。

Hbase无法再执行任何操作，包括查询和插入等，报错如下

是否能稳定复现呢？可能的原因有2个：

如果只能在hbase上复现，则可能是hbase的删除操作发送顺序有关。
如果在rocksdb上复现，则可能是schema操作的时候存在竞争条件。

暂时可以执行如下语句修复VertexLabel，将已经删除的索引从VertexLabel清理：
deletedIndexLabels = [666L, 888L];
repairedVertexLabels = [];
for (vl in graph.schema().getVertexLabels()) {
  repairVertexLabel(vl, deletedIndexLabels);
  repairedVertexLabels.add(vl);
}
repairedVertexLabels

def repairVertexLabel(vertexLabel, deletedIndexLabels) {
  for (il in vertexLabel.indexLabels()) {
    if (il.asLong() in deletedIndexLabels) {
      vertexLabel.removeIndexLabel(il);
      graph.addVertexLabel(vertexLabel);
    }
  }
  return vertexLabel;
}

authors: javeme wuchaojing

repo: hugegraph/hugegraph


text:
执行init-store.sh出现空指针异常(数据库用的cassandra)

[root@instance-futr11g9 logs]# cat hugegraph-server.log  2020-03-27 10:41:52 1771  [main] [INFO ] com.baidu.hugegraph.cmd.InitStore [] - Init graph with config file: conf/hugegraph.properties 2020-03-27 10:41:53 1937  [main] [INFO ] com.baidu.hugegraph.HugeGraph [] - Opening backend store 'cassandra' for graph 'cassandra' 2020-03-27 10:41:53 2789  [main] [WARN ] com.baidu.hugegraph.backend.store.cassandra.CassandraStore [] - Failed to close cluster after an error java.lang.NullPointerException: null 	at com.datastax.driver.core.Cluster$Manager.close(Cluster.java:1676) ~[cassandra-driver-core-3.2.0.jar:?] 	at com.datastax.driver.core.Cluster$Manager.access$200(Cluster.java:1354) ~[cassandra-driver-core-3.2.0.jar:?] 	at com.datastax.driver.core.Cluster.closeAsync(Cluster.java:566) ~[cassandra-driver-core-3.2.0.jar:?] 	at com.datastax.driver.core.Cluster.close(Cluster.java:578) ~[cassandra-driver-core-3.2.0.jar:?] 	at com.baidu.hugegraph.backend.store.cassandra.CassandraSessionPool.doClose(CassandraSessionPool.java:123) ~[hugegraph-cassandra-0.10.4.jar:?] 	at com.baidu.hugegraph.backend.store.BackendSessionPool.close(BackendSessionPool.java:140) ~[hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.backend.store.cassandra.CassandraStore.open(CassandraStore.java:154) [hugegraph-cassandra-0.10.4.jar:?] 	at com.baidu.hugegraph.backend.tx.AbstractTransaction.<init>(AbstractTransaction.java:73) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.backend.tx.IndexableTransaction.<init>(IndexableTransaction.java:30) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.backend.tx.SchemaTransaction.<init>(SchemaTransaction.java:69) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.backend.cache.CachedSchemaTransaction.<init>(CachedSchemaTransaction.java:53) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.HugeGraph.openSchemaTransaction(HugeGraph.java:258) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.HugeGraph.access$700(HugeGraph.java:80) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.HugeGraph$TinkerpopTransaction.getOrNewTransaction(HugeGraph.java:768) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.HugeGraph$TinkerpopTransaction.schemaTransaction(HugeGraph.java:747) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.HugeGraph$TinkerpopTransaction.access$000(HugeGraph.java:622) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.HugeGraph.schemaTransaction(HugeGraph.java:320) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.backend.store.BackendStoreSystemInfo.exists(BackendStoreSystemInfo.java:88) [hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.cmd.InitStore.initGraph(InitStore.java:99) [hugegraph-dist-0.10.4.jar:?] 	at com.baidu.hugegraph.cmd.InitStore.main(InitStore.java:87) [hugegraph-dist-0.10.4.jar:?] 2020-03-27 10:41:53 2794  [main] [INFO ] com.baidu.hugegraph.HugeGraph [] - Close graph hugegraph[cassandra] 2020-03-27 10:41:53 2826  [task-db-worker-1] [WARN ] com.baidu.hugegraph.backend.store.cassandra.CassandraStore [] - Failed to close cluster after an error java.lang.NullPointerException: null 	at com.datastax.driver.core.Cluster$Manager.close(Cluster.java:1676) ~[cassandra-driver-core-3.2.0.jar:?] 	at com.datastax.driver.core.Cluster$Manager.access$200(Cluster.java:1354) ~[cassandra-driver-core-3.2.0.jar:?] 	at com.datastax.driver.core.Cluster.closeAsync(Cluster.java:566) ~[cassandra-driver-core-3.2.0.jar:?] 	at com.datastax.driver.core.Cluster.close(Cluster.java:578) ~[cassandra-driver-core-3.2.0.jar:?] 	at com.baidu.hugegraph.backend.store.cassandra.CassandraSessionPool.doClose(CassandraSessionPool.java:123) ~[hugegraph-cassandra-0.10.4.jar:?] 	at com.baidu.hugegraph.backend.store.BackendSessionPool.close(BackendSessionPool.java:140) ~[hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.backend.store.cassandra.CassandraStore.open(CassandraStore.java:154) ~[hugegraph-cassandra-0.10.4.jar:?] 	at com.baidu.hugegraph.backend.tx.AbstractTransaction.<init>(AbstractTransaction.java:73) ~[hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.backend.tx.IndexableTransaction.<init>(IndexableTransaction.java:30) ~[hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.backend.tx.GraphTransaction.<init>(GraphTransaction.java:120) ~[hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.task.TaskScheduler$TaskTransaction.<init>(TaskScheduler.java:428) ~[hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.task.TaskScheduler.tx(TaskScheduler.java:115) ~[hugegraph-core-0.10.4.jar:0.10.4.0] 	at com.baidu.hugegraph.task.TaskScheduler.lambda$close$4(TaskScheduler.java:221) ~[hugegraph-core-0.10.4.jar:0.10.4.0] 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_242] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_242] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_242] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_242] 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_242]
Specifications of environment 环境信息

hugegraph-0.10.4
apache-cassandra-3.11.6

Exception in thread "main" com.baidu.hugegraph.exception.ConnectionException: Failed to connect to Cassandra 	at com.baidu.hugegraph.backend.store.cassandra.CassandraStore.open(CassandraStore.java:158) 	at com.baidu.hugegraph.backend.tx.AbstractTransaction.<init>(AbstractTransaction.java:73) 	at com.baidu.hugegraph.backend.tx.IndexableTransaction.<init>(IndexableTransaction.java:30) 	at com.baidu.hugegraph.backend.tx.SchemaTransaction.<init>(SchemaTransaction.java:69) 	at com.baidu.hugegraph.backend.cache.CachedSchemaTransaction.<init>(CachedSchemaTransaction.java:53) 	at com.baidu.hugegraph.HugeGraph.openSchemaTransaction(HugeGraph.java:258) 	at com.baidu.hugegraph.HugeGraph.access$700(HugeGraph.java:80) 	at com.baidu.hugegraph.HugeGraph$TinkerpopTransaction.getOrNewTransaction(HugeGraph.java:768) 	at com.baidu.hugegraph.HugeGraph$TinkerpopTransaction.schemaTransaction(HugeGraph.java:747) 	at com.baidu.hugegraph.HugeGraph$TinkerpopTransaction.access$000(HugeGraph.java:622) 	at com.baidu.hugegraph.HugeGraph.schemaTransaction(HugeGraph.java:320) 	at com.baidu.hugegraph.backend.store.BackendStoreSystemInfo.exists(BackendStoreSystemInfo.java:88) 	at com.baidu.hugegraph.cmd.InitStore.initGraph(InitStore.java:99) 	at com.baidu.hugegraph.cmd.InitStore.main(InitStore.java:87) Caused by: java.lang.RuntimeException: java.lang.reflect.InvocationTargetException 	at com.google.common.base.Throwables.propagate(Throwables.java:160) 	at com.datastax.driver.core.NettyUtil.newEventLoopGroupInstance(NettyUtil.java:136) 	at com.datastax.driver.core.NettyOptions.eventLoopGroup(NettyOptions.java:99) 	at com.datastax.driver.core.Connection$Factory.<init>(Connection.java:774) 	at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1446) 	at com.datastax.driver.core.Cluster.init(Cluster.java:159) 	at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:330) 	at com.datastax.driver.core.Cluster.connect(Cluster.java:280) 	at com.baidu.hugegraph.backend.store.cassandra.CassandraSessionPool$Session.open(CassandraSessionPool.java:215) 	at com.baidu.hugegraph.backend.store.cassandra.CassandraStore.open(CassandraStore.java:142) 	... 13 more
cassandra是单节点，提示不能连接，所有配置都是ok的，用java程序测试连接成功
实在不知道去哪里问了，只能在这提了
话说有没有官方QQ群之类的


text:
Can we use Multi-LORA CPU

Hi,
Im currently following this doc: https://huggingface.co/docs/google-cloud/en/examples/gke-tgi-multi-lora-deployment
After got a bug: "Can’t scale up due to exceeded quota" and do some research, I suspect that my free trial (300$) account is not able to increase GPU quota (even I have activated my account to not be trial anymore and have to contact sale)
Is there anyway I can run this with cpu instead.
Thank you

Hi here @AndrewNgo-ini I'm afraid you won't be able to run TGI on CPUs with the current container on Google Cloud, as that's only for GPU (and coming for TPU too).
Anyway, you should be able to run TGI on Intel CPUs as of https://huggingface.co/docs/text-generation-inference/installation_intel#using-tgi-with-intel-cpus, even if it's not the recommended hardware. You should be able to re-use the container at ghcr.io/huggingface/text-generation-inference:2.4.1-intel-cpu, let me know if that works.
Hope that helps 🤗 Also happy to know more about your use-case / needs to see how we can support those better in the coming months!

Good day @alvarobartt
My use case is to demo for devfest day of GDG. so latency wouldnt be a problem.
I’ll try to try out cpu approach and would not mind to have a PR to adjust this problem. Any suggestion I can do this properly

Great, so atm I'm afraid that we're not shipping the Text Generation Inference (TGI) DLC for any other hardware than NVIDIA GPUs and soon to come Google TPUs, so you could still demo it, but without the container being officially hosted on Google Cloud; anyway, let me know if you run into any issues when running the demo, happy to help! 🤗

authors: AndrewNgo-ini alvarobartt

repo: huggingface/Google-Cloud-Containers


text:
Contigous on gather

Solves https://github.com/huggingface/transformers/issues/25076
Solves https://github.com/huggingface/accelerate/issues/1764
We need to ensure tensors are contiguous before gathering, else torch will raise an error.

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint.

authors: HuggingFaceDocBuilderDev muellerzr

repo: huggingface/accelerate


text:
Adding requires_grad to kwargs when registering empty parameters.

What does this PR do?

Fixes #2361
Before submitting

[ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
[x] Did you read the contributor guideline,
Pull Request section?
[ ] Was this discussed/approved via a Github issue or the forum? Please add a link
to it if that's the case.
[x] Did you make sure to update the documentation with your changes? Here are the
documentation guidelines, and
here are tips on formatting docstrings.
[ ] Did you write any new necessary tests?

Who can review?
Anyone in the community is free to review the PR once the tests have passed. Feel free to tag
members/contributors who may be interested in your PR.

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

@younesbelkada  Looks like no problems there:
CUDA_VISIBLE_DEVICES=7 RUN_SLOW=1 pytest tests/quantization/bnb/test_4bit.py
==================================================================================================================================== test session starts =====================================================================================================================================
platform linux -- Python 3.11.5, pytest-7.4.4, pluggy-1.4.0
rootdir: /home/blacksamorez/transformers
configfile: pyproject.toml
plugins: anyio-3.7.1
collected 37 items

tests/quantization/bnb/test_4bit.py ..............s......................                                                                                [100%]

====================================================================================================================================== warnings summary ======================================================================================================================================
../quip-sharp/.conda/lib/python3.11/site-packages/_pytest/config/__init__.py:1373
  /home/blacksamorez/quip-sharp/.conda/lib/python3.11/site-packages/_pytest/config/__init__.py:1373: PytestConfigWarning: Unknown config option: doctest_glob

    self._warn_or_fail_if_strict(f"Unknown config option: {key}\n")

tests/quantization/bnb/test_4bit.py: 10 warnings
  /home/blacksamorez/quip-sharp/.conda/lib/python3.11/site-packages/bitsandbytes/nn/modules.py:226: UserWarning: Input type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference or training speed.
    warnings.warn(f'Input type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference or training speed.')

tests/quantization/bnb/test_4bit.py::Bnb4BitTest::test_rwkv_4bit
  /home/blacksamorez/quip-sharp/.conda/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
    return self.fget.__get__(instance, owner)()

tests/quantization/bnb/test_4bit.py::Bnb4BitTest::test_rwkv_4bit
  /home/blacksamorez/quip-sharp/.conda/lib/python3.11/site-packages/torch/utils/cpp_extension.py:28: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html
    from pkg_resources import packaging  # type: ignore[attr-defined]

tests/quantization/bnb/test_4bit.py::Bnb4BitT5Test::test_inference_with_keep_in_fp32
tests/quantization/bnb/test_4bit.py::Bnb4BitT5Test::test_inference_with_keep_in_fp32
tests/quantization/bnb/test_4bit.py::Bnb4BitT5Test::test_inference_without_keep_in_fp32
tests/quantization/bnb/test_4bit.py::Bnb4BitT5Test::test_inference_without_keep_in_fp32
  /home/blacksamorez/transformers/src/transformers/generation/utils.py:1133: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.
    warnings.warn(

tests/quantization/bnb/test_4bit.py::Bnb4BitT5Test::test_inference_with_keep_in_fp32
tests/quantization/bnb/test_4bit.py::Bnb4BitT5Test::test_inference_without_keep_in_fp32
  /home/blacksamorez/quip-sharp/.conda/lib/python3.11/site-packages/bitsandbytes/nn/modules.py:223: UserWarning: Input type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference.
    warnings.warn(f'Input type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference.')

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=================================================================================================================== 36 passed, 1 skipped, 19 warnings in 344.18s (0:05:44) ===================================================================================================================

Setup:
Name: accelerate
Version: 0.27.0.dev0
Summary: Accelerate
Home-page: https://github.com/huggingface/accelerate
Author: The HuggingFace team
Author-email: sylvain@huggingface.co
License: Apache
Location: /home/blacksamorez/quip-sharp/.conda/lib/python3.11/site-packages
Editable project location: /home/blacksamorez/accelerate
Requires: huggingface-hub, numpy, packaging, psutil, pyyaml, safetensors, torch
Required-by:
---
Name: transformers
Version: 4.38.0.dev0
Summary: State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow
Home-page: https://github.com/huggingface/transformers
Author: The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)
Author-email: transformers@huggingface.co
License: Apache 2.0 License
Location: /home/blacksamorez/quip-sharp/.conda/lib/python3.11/site-packages
Editable project location: /home/blacksamorez/transformers
Requires: filelock, huggingface-hub, numpy, packaging, pyyaml, regex, requests, safetensors, tokenizers, tqdm
Required-by: lm-eval
---
Name: bitsandbytes
Version: 0.42.0
Summary: k-bit optimizers and matrix multiplication routines.
Home-page: https://github.com/TimDettmers/bitsandbytes
Author: Tim Dettmers
Author-email: dettmers@cs.washington.edu
License: MIT
Location: /home/blacksamorez/quip-sharp/.conda/lib/python3.11/site-packages
Requires: scipy
Required-by:

authors: BlackSamorez HuggingFaceDocBuilderDev


text:
Einops-like API

Having an API similar to rearrange, reduce, and repeat from einops would be nice.

Just saw that there was a community crate doing this.

repo: huggingface/candle


text:
Notebook: unstructured data preprocessing for RAG

The PR adds a notebook illustrating how to preprocess a few different types of unstructured data into format that can be loaded into a vector store and used for RAG with LangChain.

Ping @merveenoyan @stevhliu 🤗

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

Thank you @stevhliu ! I addressed your feedback :)

@MKhalusova always a pleasure to work with you <3 added mostly nits

Thanks so much @merveenoyan! I addressed everything except the last comment  - I instead give a link at the very beginning to a notebook that explains the building blocks of RAG, to avoid making it too long.

Hi @stevhliu ! Can you help me resolve the conflict in TOC? I'm not sure what structure you want to have there.
Other than that, looks like the PR is ready :)

@MKhalusova we recently added enterprise recipes in https://huggingface.co/learn/cookbook/index so you can just use the most recent toctree and add your recipe under the LLM recipes

@merveenoyan Done!

repo: huggingface/cookbook


text:
Change toctree

This PR divides toctree and removes a few recipe links in "new recipes" in index since they were getting chunky

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

authors: HuggingFaceDocBuilderDev merveenoyan


text:
Fix CI test StorageClient.url_preparator

Fix CI test StorageClient.url_preparator.
This fail in CI tests was introduced after:

#2966

See: https://github.com/huggingface/dataset-viewer/actions/runs/10350752694/job/28648303009?pr=3018
FAILED tests/test_response.py::test_create_response_with_image - AssertionError: assert [{'row': {'im...d_cells': []}] == [{'row': {'im...d_cells': []}]

  At index 0 diff: {'row_idx': 0, 'row': {'image': {'src': 'http://localhost/cached-assets/ds_image/--/{dataset_git_revision}/--/default/train/0/image/image.jpg', 'height': 480, 'width': 640}}, 'truncated_cells': []} != {'row_idx': 0, 'row': {'image': {'src': 'http://localhost/cached-assets/ds_image/--/revision/--/default/train/0/image/image.jpg', 'height': 480, 'width': 640}}, 'truncated_cells': []}

  Full diff:
    [
        {
            'row': {
                'image': {
                    'height': 480,
  -                 'src': 'http://localhost/cached-assets/ds_image/--/revision/--/default/train/0/image/image.jpg',
  +                 'src': 'http://localhost/cached-assets/ds_image/--/{dataset_git_revision}/--/default/train/0/image/image.jpg',
  ?                                                                    +++++++++++++        +
                    'width': 640,
                },
            },
            'row_idx': 0,
            'truncated_cells': [],
        },
    ]

The error in the CI is unrelated to this PR.
See:

#3021
#3022

repo: huggingface/dataset-viewer


text:
News_commentary Dataset Translation Pairs are of Incorrect Language Specified Pairs

I used load_dataset to load the news_commentary dataset for "ar-en" translation pairs but found translations from Arabic to Hindi.
train_ds = load_dataset("news_commentary", "ar-en", split='train[:98%]')
val_ds = load_dataset("news_commentary", "ar-en", split='train[98%:]')

# filtering out examples that are not ar-en translations but ar-hi
val_ds = val_ds.filter(lambda example, indice: indice not in chain(range(1312,1327) ,range(1384,1399), range(1030,1042)), with_indices=True)

I'm fairly new to using datasets so I might be doing something wrong

These ranges seem to be valid English. Closing.

authors: anassalamah mariosasko

repo: huggingface/datasets


text:
Remove compression from xopen

We implemented support for streaming with 2 requirements:

transparent use for the end user: just needs to pass the parameter streaming=True
no additional work for the contributors: previous loading scripts should also work in streaming mode with no (or minor) changes; and new loading scripts should not involve additional code to support streaming

In order to fulfill these requirements, streaming implementation patched some Python functions:

the open(urlpath) function was patched with fsspec.open(urlpath)
the os.path.join(urlpath, *others) function was patched in order to add to urlpath hops (::) and extractor protocols (zip://), which are required by fsspec.open

Recently, we implemented support for streaming all archive+compression formats: zip, tar, gz, bz2, lz4, xz, zst; tar.gz, tar.bz2,...
Under the hood, the implementation:

passes an additional parameter compression to fsspec.open, so that it performs the decompression on the fly: fsspec.open(urlpath, compression=...)

Some concerns have been raised about passing the parameter compression to fsspec.open:

https://github.com/huggingface/datasets/pull/2786#discussion_r689550254
#2811

The main argument is that if open decompresses the file and afterwards we call gzip.open on it, that will raise an error in oscar dataset:
gzip.open(open(urlpath

While this is true:

it is not natural/usual to call open inside gzip.open (never seen this before)
indeed, this was recently (2 months ago) coded that way in datasets in order to allow streaming support (with previous implementation of streaming)

In this particular case, there is a natural fix solution: #2811:

Revert the open inside the gzip.open (change done 2 months ago): gzip.open(open(urlpath => gzip.open(urlpath
Patch gzip.open(urlpath with fsspec.open(urlpath, compression="gzip"

Are there other issues apart from this?
Note that there is an issue just because the open inside of the gzip.open. There is no issue in the other cases where datasets loading scripts use just

gzip.open
open (after having called dl_manager.download_and_extract)

TODO:

[ ] Is this really an issue? Please enumerate the datasets loading scripts where this is problematic.

For the moment, there are only 3 datasets where we have an open inside a gzip.open:

oscar (since 23 June), mc4 (since 2 July) and c4 (since 2 July)

In the 3 datasets, the only reason to put an open inside a gzip.open was indeed to force supporting streaming

[ ] If this is indeed an issue, which are the possible alternatives? Pros/cons?

After discussing with @lhoestq, a reasonable alternative:

download_manager.extract(urlpath adds prefixes to urlpath in the same way as fsspec does for protocols, but we add all compression formats
xopen parses the urlpath and extracts the compression parameter and passes it to fsspec.open


text:
TypeError: 'bool' object is not iterable when filtering a datasets.arrow_dataset.Dataset

Describe the bug
Hi,
Thanks for the amazing work on the library!
Describe the bug
I think I might have noticed a small bug in the filter method.
Having loaded a dataset using load_dataset,  when I try to filter out empty entries with batched=True, I get a TypeError.
Steps to reproduce the bug
train_dataset = train_dataset.filter(
            function=lambda example: example["image"] is not None,
            batched=True,
            batch_size=10)

Error message:
File .../lib/python3.9/site-packages/datasets/fingerprint.py:480, in fingerprint_transform.<locals>._fingerprint.<locals>.wrapper(*args, **kwargs)
    476             validate_fingerprint(kwargs[fingerprint_name])
    478 # Call actual function
--> 480 out = func(self, *args, **kwargs)
...
-> 5666 indices_array = [i for i, to_keep in zip(indices, mask) if to_keep]
   5667 if indices_mapping is not None:
   5668     indices_array = pa.array(indices_array, type=pa.uint64())

TypeError: 'bool' object is not iterable

Removing batched=True allows to bypass the issue.
Expected behavior
According to the doc, "[batch_size corresponds to the] number of examples per batch provided to function if batched = True", so we shouldn't need to remove the batchd=True arg?
source: https://huggingface.co/docs/datasets/v2.9.0/en/package_reference/main_classes#datasets.Dataset.filter
Environment info

datasets version: 2.9.0
Platform: Linux-5.4.0-122-generic-x86_64-with-glibc2.31
Python version: 3.9.10
PyArrow version: 10.0.1
Pandas version: 1.5.3

Thank a lot!

I hit the same issue and the error message isn't really clear on what's going wrong. It might be helpful to update the docs with a batched example.


text:
Make dataset viewer more flexible in displaying metadata alongside images

Feature request
To display images with their associated metadata in the dataset viewer, a metadata.csv file is required. In the case of a dataset with multiple subsets, this would require the CSVs to be contained in the same folder as the images since they all need to be named metadata.csv. The request is that this be made more flexible for datasets with multiple subsets to avoid the need to put a metadata.csv into each image directory where they are not as easily accessed.
Motivation
When creating datasets with multiple subsets I can't get the images to display alongside their associated metadata (it's usually one or the other that will show up). Since this requires a file specifically named metadata.csv, I then have to place that file within the image directory, which makes it much more difficult to access. Additionally, it still doesn't necessarily display the images alongside their metadata correctly (see, for instance, this discussion).
It was suggested I bring this discussion to GitHub on another dataset struggling with a similar issue (discussion). In that case, it's a mix of data subsets, where some just reference the image URLs, while others actually have the images uploaded. The ones with images uploaded are not displaying images, but renaming that file to just metadata.csv would diminish the clarity of the construction of the dataset itself (and I'm not entirely convinced it would solve the issue).
Your contribution
I can make a suggestion for one approach to address the issue:
For instance, even if it could just end in _metadata.csv or -metadata.csv, that would be very helpful to allow for more flexibility of dataset structure without impacting clarity. I would think that the functionality on the backend looking for metadata.csv could reasonably be adapted to look for such an ending on a filename (maybe also check that it has a file_name column?).
Presumably, requiring the configs in a setup like on this dataset could also help in figuring out how it should work?
configs:
  - config_name: <image subset>
    data_files:
      - <image-metadata>.csv
      - <path/to/images>/*.jpg

I'd also be happy to look at whatever solution is decided upon and contribute to the ideation.
Thanks for your time and consideration! The dataset viewer really is fabulous when it works :)

Note that you can already have one directory per subset just for the metadata, e.g.
configs:
  - config_name: subset0
    data_files:
      - subset0/metadata.csv
      - images/*.jpg
  - config_name: subset1
    data_files:
      - subset1/metadata.csv
      - images/*.jpg

I think supporting **/-metadata.csv or **/_metadata.csv makes sense to me. If it sounds good to you feel free to open a PR to update the patterns here:
https://github.com/huggingface/datasets/blob/d4422cc24a56dc7132ddc3fd6b285c5edbd60b8c/src/datasets/data_files.py#L104-L115


text:
Make decoding of Audio and Image feature optional

Add the decode argument (True by default) to the Audio and the Image feature to make it possible to toggle on/off decoding of these features.
Even though we've discussed that on Slack, I'm not removing the _storage_dtype argument of the Audio feature in this PR to avoid breaking the Audio feature tests.

Closing this PR for now due to https://github.com/huggingface/datasets/issues/3145#issuecomment-993664104.

Okay, after some more thinking, I'm re-opening this PR for three reasons:

This feature will allow us to remove the image_file_path/audio_file_path columns in our vision/audio datasets. Currently, it makes sense to keep those columns because it's not obvious how to access the underlying path information of the Image/Audio feature. However, if the user is not aware and does dataset[0]["image_file_path"] on our vision/audio datasets, this will be costly because the image/audio file data has to be decoded first (stored in dataset[0]["image"]/dataset[0]["audio"])
In CV, we often work with the so-called "half life" datasets (RedCaps, WIT, ...) that only provide image URLs and not actual image data, and some of these image URLs even go down. Our solution to this problem is to give a note on how to efficiently download the image data using map in the datasets cards of such datasets. This feature will remove the need for a separate image_url column of type Value("string") in such datasets. Instead, we will be able to use the image column of type Image()  (the image feature knows how to decode image URLs using xopen), disable decoding and use requests.get for download, which I expect to be faster than xopen.
This feature should help us in implementing push_to_hub for the Image/Audio where we transfer actual image/audio data and not paths

This feature will allow us to remove the image_file_path/audio_file_path columns in our vision/audio datasets. Currently, it makes sense to keep those columns because it's not obvious how to access the underlying path information of the Image/Audio feature. However, if the user is not aware and does dataset[0]["image_file_path"] on our vision/audio datasets, this will be costly because the image/audio file data has to be decoded first (stored in dataset[0]["image"]/dataset[0]["audio"])

That makes sense !

Instead, we will be able to use the image column of type Image() (the image feature knows how to decode image URLs using xopen), disable decoding and use requests.get for download, which I expect to be faster than xopen.

I feel like it's a bit convoluted compared to having the image_url column as string, and say to users to map using requests.get with image_url.
Moreover I'm not 100% sure that we should have Image features with both local paths and URLs, since this behavior is a bit hidden the users and they don't give the same performance at all.

This feature should help us in implementing push_to_hub for the Image/Audio where we transfer actual image/audio data and not paths

Cool !

Do you mean we should remove support for URLs in the Image feature? Because this is what we get for free by adding support for streaming (by using xopen instead of open) and this is also what the Audio feature does.

I think it might not be ideal to have URLs in an Image type column for a dataset in non-streaming mode, since you'd expect to have everything locally. But for a streaming dataset it must use xopen indeed

Yes, I agree. Let's have the image_url columns as Value("string") + a note with the map function to download images for local datasets and a note with cast_column (which is requested https://github.com/huggingface/datasets/issues/3369) for streamed datasets (ds.cast_column("image_url", Image())).

I fixed the merge conflicts and small bugs in nested decoding introduced by #3575. Additionally, I addressed https://github.com/huggingface/datasets/issues/3473 by adding the _iter method to Dataset (inspired by the _getitem method). For iter(dset) I set _iter(dset, decoded=True) to enable decoding and for map _iter(dset, decoded=False) to make it lazy.

authors: lhoestq mariosasko


text:
[BUG] how to eval large scale model use 1dp+8pp?

Describe the bug
I tired to eval a large scale model use1dp+8pp with accelerate. I use the command like the following:
accelerate launch --multi_gpu --num_processes=1 run_evals_accelerate.py \
    --model_args="pretrained=<path to model on the hub>" \
    --model_parallel \
    --tasks <task parameters> \
    --output_dir output_dir

The error is ValueError: You need to use at least 2 processes to use --multi_gpu
How to solve this problem?
Version info
lighteval-0.3.0

https://github.com/huggingface/lighteval/blob/v0.3.0/README.md#evaluating-a-large-model-with-pipeline-parallelism
I use an old version. Does it support DP = 1 + PP=K?

I updated lighteval to V0.6.0 and tried again.
I used the following command:
export CUDA_VISIBLE_DEVICES=0,1,2,3
lighteval accelerate "pretrained=<model_path>,model_parallel=True" "leaderboard|truthfulqa:mc|0|0"

It failed to load the model with MP, and only GPU 0 was occupied.
[2024-12-25 03:29:43,045] [    INFO]: Model parallel was set to False, max memory set to None and device map to None (base_model.py:389)

I found 'LOCAL_WORLD_SIZE' and 'WORLD_SIZE' was none.
How to solve this problem?

Hi ! You are right, I made opened a PR you can checkout it should solve the issue.

authors: NathanHB mxjmtxrm

repo: huggingface/lighteval


text:
Small test_text_generation_example.py refacto

What does this PR do?

This PR moves the env variables declaration into a dedicated dictionary for each test. This ensures that env variables that are declared for one test are not declared anymore for the following tests.
Before submitting

[ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
[ ] Did you make sure to update the documentation with your changes?
[ ] Did you write any new necessary tests?

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

authors: HuggingFaceDocBuilderDev regisss

repo: huggingface/optimum-habana


text:
support assisted decoding in ipex 2.5

Support assisted decoding if ipex >= 2.5

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint. The docs are available until 30 days after the last update.

Hi @echarlaix . I have fixed all your comments, please re-run the tests. Thx!

Hi @IlyasMoutawwakil , could you please review this PR? Thx!

Can you please add an explanation in the PR of why this patching is necessary ? Do assistant models interfere with torchscript ?

Can you please add an explanation in the PR of why this patching is necessary ? Do assistant models interfere with torchscript ?

Yes. In assisted decoding, both the target model and the assistant model need to call crop_pask_key_values. The patched IPEXModel has a different format of past_key_values, so we need our own crop functions to apply IPEX KV-cache.

Yes. In assisted decoding, both the target model and the assistant model need to call crop_pask_key_values. The patched IPEXModel has a different format of past_key_values, so we need our own crop functions to apply IPEX KV-cache.
Do you want me to write it as a comment in the codes?

Thanks for the explanation ! One last code change and it's good for me 🤗

Yes. In assisted decoding, both the target model and the assistant model need to call crop_pask_key_values. The patched IPEXModel has a different format of past_key_values, so we need our own crop functions to apply IPEX KV-cache.
Do you want me to write it as a comment in the codes?

Thanks for the explanation ! One last code change and it's good for me 🤗

I have fixed your required changes. The _unpatch_crop_past_key_values() can be safely added at the end the generation because it will revert changes if we do patch and keep the same if we didn't do patch.

Hi @echarlaix . I have enabled assisted decoding tests in ipex 2.4. Please take a review, thx~

Hi @echarlaix . I can do the unpatch for target model only, please look the new changes

repo: huggingface/optimum-intel


text:
Accessing Base model after Lora addition

Hi,
Thanks for your great project.
I have one question, can we somehow access the base model after we add a Lora model adapter?
This is really useful in case we need to apply inference and get the results of the base model and at the same time be able to get the results from the lora adapter model.

Hello @agemagician
I think you can leverage the disable_adapter context manager, check an example here: https://github.com/lvwerra/trl/blob/03d98447300efc6bfd14cc271fdbfe175973e9ca/trl/trainer/ppo_trainer.py#L451
cc @pacman100

This is really great and will work for sequential model accessing.
However, if this model is accessed from different threads, results might be overloaded between enabled and disabled.
I think having it as an input to the forward function is better, so it would be independent on each multiple call.

For now, this is a good solution for me :)
Feel free to close it :)

@younesbelkada Hi, is it possible to change lora weights after loading? For instance, I have 2 lora weights, L1 & L2, after merge with L1, I may want to unload L1 and merge with L2 so that I don't need to reload the base model. Is that possible now for peft?

hi @edwardelric1202 thanks for the ping!
You mean merging 2 lora weights on the base model?
If that's the case you can load the first LoRA weight using load_adapter method, merge it and save the merged model as a standalone model, then load the L2 weights in that new model. What do you think?

@younesbelkada Thx for your reply. Sry for didn't make it clear, in fact I wonder if it is possible to unmerge L1 after merge with it, so that the model will return to the original one, then I can merge it with L2? It means I can change different Lora adapter without reloading the base model.

I see better now thanks for clarifying, as for now i don't think this i supported unfortunately and I am not sure we will support it in the future as it sounds like a very edge usecase

Got it, thanks for your reply!

repo: huggingface/peft


text:
Dataset was loaded twice in 4-bit finetuning script

Was working on something unrelated and saw that the dataset was always loaded twice in the finetuning script so I removed the double occurence.

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint.

authors: HuggingFaceDocBuilderDev lukaskuhn-lku


text:
load incorrect weight values from a HF model on a big-endian machine

System Info
Python: 3.10
PyTorch: the latest main branch (i.e. 2.0.1+)
safetensor: 0.3.1
Platform: s390x (big-endian)

Information

[ ] The official example scripts
[X] My own modified scripts

Reproduction
Execute the following script on s390x after installing safetensor.
from transformers import AutoModelForSequenceClassification
torch.serialization.set_default_load_endianness(torch.serialization.LoadEndianness.LITTLE)
model = AutoModelForSequenceClassification.from_pretrained("roberta-base")
print(model.roberta.embeddings.position_embeddings.weight)

The output is
Parameter containing:
tensor([[1.4715e-38, 1.7692e-38, 5.9353e-39,  ..., 8.8744e-39, 3.0194e-39,
         1.4712e-38],
        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00, 0.0000e+00,
         0.0000e+00],
        [1.7637e-38, 1.7682e-38, 8.8821e-39,  ..., 2.9727e-39, 1.4779e-38,
         5.1024e-41],
        ...,
        [2.0658e-38, 1.4734e-38, 8.8287e-39,  ..., 5.2997e-41, 2.0663e-38,
         6.0351e-41],
        [1.4755e-38, 1.7712e-38, 2.0579e-38,  ..., 1.7650e-38, 2.0572e-38,
         5.9387e-39],
        [3.0012e-39, 1.7679e-38, 1.7633e-38,  ..., 2.0641e-38, 1.7639e-38,
         8.8477e-39]], requires_grad=True)

Expected behavior
When I tried to execute an HF transformer model (i.e. little endian layout) with safetensor format on a big-endian machine, I met this issue.
When I execute the same script on s390x w/o installing safetensor.
from transformers import AutoModelForSequenceClassification
torch.serialization.set_default_load_endianness(torch.serialization.LoadEndianness.LITTLE)
model = AutoModelForSequenceClassification.from_pretrained("roberta-base")
print(model.roberta.embeddings.position_embeddings.weight)

The output is
Parameter containing:
tensor([[-0.0115,  0.0204,  0.0197,  ...,  0.0050, -0.0274, -0.0439],
        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],
        [ 0.0346, -0.0169, -0.0895,  ..., -0.0542,  0.0291,  0.0173],
        ...,
        [ 0.1191, -0.0587, -0.0396,  ..., -0.0179,  0.1249,  0.0205],
        [ 0.0833, -0.0272, -0.0368,  ..., -0.0429,  0.1288, -0.0052],
        [ 0.0850,  0.2573, -0.1257,  ..., -0.0952,  0.1443, -0.0131]],
       requires_grad=True)

I expect this result even using safetensor format.

First time I'm encountering a big endian machine.
For torch this lib is using https://pytorch.org/docs/stable/storage.html?highlight=from+storage#torch.UntypedStorage.from_file which allows for fast loading.
I don't seen any flags to specify endianness.
There are currently some protection to prevent writing a big endian file (for numpy).
I must say I'm not sure how torch handles endianness.
https://github.com/pytorch/pytorch/issues/101688
Would you happen to know any way I could emulate bigendianness or get easy access to such a machine ? (So I could fix those holes)

This is a good document to get an access to s390x.

To write a big endian file for pytorch is also prohibited.
$ cat st.py
import safetensors
from safetensors.torch import save_file, load_file
import torch

tensors = {
   "a": torch.zeros((2, 3)),
   "b": torch.zeros((4, 5), dtype=torch.int16)
}

save_file(tensors, "./model.safetensors.s390x")

# Now loading
loaded = load_file("./model.safetensors.s390x")
print(loaded)
$ python st.py
Traceback (most recent call last):
  File "/home/ishizaki/PyTorch/tmp/st.py", line 10, in <module>
    save_file(tensors, "./model.safetensors.s390x")
  File "/home/ishizaki/PyTorch/.venv/lib/python3.10/site-packages/safetensors/torch.py", line 232, in save_file
    serialize_file(_flatten(tensors), filename, metadata=metadata)
  File "/home/ishizaki/PyTorch/.venv/lib/python3.10/site-packages/safetensors/torch.py", line 377, in _flatten
    raise ValueError("Big endian is not supported, serialization need to be in little endian")
ValueError: Big endian is not supported, serialization need to be in little endian

Would you happen to know any way I could emulate bigendianness or get easy access to such a machine ? (So I could fix those holes)

Another relatively easy way should probably be qemu-system-s390x with a Debian s390 live CD :)

I will take a one-week vacation from Friday. I will respond slowly for a while.

Now, I have been back from my vacation. I executed these two programs with 0.3.2. However, the results are not what I expected. Since I cannot reopen this issue, I will open a new issue and investigate the reason.

authors: Narsil akx kiszk

repo: huggingface/safetensors


text:
Allow LM selection and MLX Gemma

Allow the language model to be selected via command line when running on Mac with MLX.
Remove the system message if using Gemma (otherwise the chat templating will throw an error).

Cool stuff! The only issue I see is that the default model in the language model won't work for MLX, so we would need to change that.

I'll merge this and make that change quickly

great thanks
On Fri, Aug 23, 2024 at 1:06 PM Andrés Marafioti @.***>
wrote:

Merged #40 https://github.com/huggingface/speech-to-speech/pull/40 into
main.
—
Reply to this email directly, view it on GitHub
https://github.com/huggingface/speech-to-speech/pull/40#event-13992357299,
or unsubscribe
https://github.com/notifications/unsubscribe-auth/ASVG6CSTG7ORPSGPM2XFFQ3ZS4QUPAVCNFSM6AAAAABM76KSPKVHI2DSMVQWIX3LMV45UABCJFZXG5LFIV3GK3TUJZXXI2LGNFRWC5DJN5XDWMJTHE4TEMZVG4ZDSOI
.
You are receiving this because you authored the thread.Message ID:
@.***>

authors: RonanKMcGovern andimarafioti

repo: huggingface/speech-to-speech


text:
why would the cpu-0.6 try to use gpu ?

System Info
docker image built for cpu
Information

[X] Docker
[ ] The CLI directly

Tasks

[ ] An officially supported command
[ ] My own modifications

Reproduction
I try the simplest command to start the cpu version as below
docker run --gpus all -p 8888:80 -v $volume:/data -d ghcr.io/huggingface/text-embeddings-inference:cpu-0.6 --model-id $model --revision $revision
but get error below
docker: Error response from daemon: could not select device driver "" with capabilities: [[gpu]].
this is a machine without gpu card
Expected behavior
it should start

Remove --gpus all from the docker command.

repo: huggingface/text-embeddings-inference


text:
Flash attention is not installed.

I run this:
text-embeddings-router --model-id Salesforce/SFR-Embedding-Mistral --port 8080 --dtype float16
Then I get the following error:

I have installed flash-attn:

The same error when I run: text-embeddings-router --model-id Salesforce/SFR-Embedding-2_R --port 8080 --dtype float16
What's wrong?

How did you install TEI?
Did you install it with cuda support?
Do you have a supported GPU (cuda capability > 7.5)?

How did you install TEI? Did you install it with cuda support? Do you have a supported GPU (cuda capability > 7.5)?

I install TEI with your step CUDA: cargo install --path router -F candle-cuda-turing -F http --no-default-features, my CUDA version is 12.4 and also add the nvidia binaries to path. My GPU is 3080, is it enough?

Use cargo install --path router -F candle-cuda -F http --no-default-features instead.
Note the candle-cuda instead of candle-cuda-turing.
A 3080 is not a turing GPU but an Ampere one.

The candle-cuda-turing feature should only be used for old GPUs like T4s.
Also, 1.3 had a bug for mistral. I strongly advise you to update either to latest or 1.4.

Okay, I will try 1.4 via cargo install --path router -F candle-cuda -F http --no-default-features, thanks a lot


text:
Added Pending / Success / Failed counts #266

I use localStorage to store local the total number and count success and failure snapshots and uploads. If these values need to be stored on the server-side then we should use rest API to get and post the values.

snapshots and uploads are arrays and you should be able to count them directly from the array using a computed property for each. For example,
computed: {   failedUploadCount: function () {     return this.uploads.filter( u => { return u.status === 'FAILED' }).length   }
Add the computed properties for each field and then display it.

snapshots and uploads are arrays and you should be able to count them directly from the array using a computed property for each. For example,
computed: {   failedUploadCount: function () {     return this.uploads.filter( u => { return u.status === 'FAILED' }).length   }
Add the computed properties for each field and then display it.

computed: {
uploadsFiltered: function () {
let returnArray = this.uploads
if (this.search && this.search.length > 0) {
returnArray = this.uploads.filter(u => {
return ((!u.device_name) || u.device_name.toLowerCase().indexOf(this.search.toLowerCase()) > -1)
})
}
return returnArray
},
snapshotSuccessCount: function () {
return this.snapshots.filter(s => s.status === 'SUCCESS').length
},
snapshotFailedCount: function () {
return this.snapshots.filter(s => s.status === 'FAILED').length
},
snapshotPendingCount: function () {
return this.snapshots.filter(s => s.status === 'PENDING').length
},
uploadsSuccessCount: function () {
return this.uploads.filter(s => s.status === 'SUCCESS').length
},
uploadsFailedCount: function () {
return this.uploads.filter(s => s.status === 'FAILED').length
},
uploadsPendingCount: function () {
return this.uploads.filter(s => s.status === 'PENDING').length
}
},

authors: disperse olgamani

repo: human-nature-lab/trellis-app


text:
Images in feed need full url + alt text

Just like the main_image

repo: human-nyc/spree-blogging-spree