text:
Index out of range

I've hit what appears to be a bug in ragel. As part of a larger machine, I've built a string parser that interprets "bare" strings (i.e. not enclosed within quotes).
The two-character start sequence check exists because there are valid sequences that begin with a character, followed by a symbol (such as u"). The rest of the machine is omitted from this test code for brevity.
The problem is that the generated code causes an index-out-of-bounds error. I've distilled the issue down to this test code:
decoder.rl

package ragelbug

import (
    "fmt"
)

%%{
    machine ragelbug;

    ws = [\r\n\t ];

    unquoted_first_char = [A-Za-z_];

    unquoted_second_char = unquoted_first_char | [0-9];

    unquoted_remaining_char = [0-9A-Za-z_];

    unquoted_string = unquoted_first_char unquoted_second_char @{
        fcall unquoted_string_iterate;
    };

    unquoted_string_iterate := unquoted_remaining_char* %{
        if err = callbacks.OnArrayData(data[arrayStart:fpc]); err != nil {
            fbreak;
        }
        if err = callbacks.OnArrayEnd(); err != nil {
            fbreak;
        }
        firstTwoCharacters = firstTwoCharacters[:0]
        fret;
    };

    main := unquoted_string ws* @/{
        err = fmt.Errorf("Incomplete document")
    };
}%%

%% write data;

type ArrayType int
const (
    ArrayTypeBinary = ArrayType(iota)
    ArrayTypeString
)

type DecoderCallbacks interface {
    OnArrayBegin(arrayType ArrayType) error
    OnArrayData(bytes []byte) error
    OnArrayEnd() error
}

func Parse(data []byte, callbacks DecoderCallbacks) (err error) {
    cs := 0
    p := 0
    pe := len(data)
    eof := pe
    top := 0
    stack := make([]int, 100)
    firstTwoCharacters := make([]byte, 0, 8)
    arrayStart := 0
    _ = eof

    %%{
        write init;
        write exec;
    }%%

    if cs == ragelbug_error {
        err = fmt.Errorf("Parse error at %v", p)
    }

    return
}

decoder.go (generated)

//line decoder.rl:1
package ragelbug

import (
    "fmt"
)

//line decoder.rl:36

//line decoder.go:16
var _ragelbug_actions []byte = []byte{
	0, 1, 0, 1, 1,
}

var _ragelbug_key_offsets []byte = []byte{
	0, 0, 5, 12, 16,
}

var _ragelbug_trans_keys []byte = []byte{
	95, 65, 90, 97, 122, 95, 48, 57,
	65, 90, 97, 122, 13, 32, 9, 10,
	95, 48, 57, 65, 90, 97, 122,
}

var _ragelbug_single_lengths []byte = []byte{
	0, 1, 1, 2, 1,
}

var _ragelbug_range_lengths []byte = []byte{
	0, 2, 3, 1, 3,
}

var _ragelbug_index_offsets []byte = []byte{
	0, 0, 4, 9, 13,
}

var _ragelbug_indicies []byte = []byte{
	0, 0, 0, 1, 2, 2, 2, 2,
	1, 3, 3, 3, 1, 4, 4, 4,
	4, 1,
}

var _ragelbug_trans_targs []byte = []byte{
	2, 0, 3, 3, 4,
}

var _ragelbug_trans_actions []byte = []byte{
	0, 0, 1, 0, 0,
}

var _ragelbug_eof_actions []byte = []byte{
	0, 0, 0, 0, 3,
}

const ragelbug_start int = 1
const ragelbug_first_final int = 3
const ragelbug_error int = 0

const ragelbug_en_unquoted_string_iterate int = 4
const ragelbug_en_main int = 1

//line decoder.rl:40

type ArrayType int
const (
    ArrayTypeBinary = ArrayType(iota)
    ArrayTypeString
)

type DecoderCallbacks interface {
    OnArrayBegin(arrayType ArrayType) error
    OnArrayData(bytes []byte) error
    OnArrayEnd() error
}

func Parse(data []byte, callbacks DecoderCallbacks) (err error) {
    cs := 0
    p := 0
    pe := len(data)
    eof := pe
    top := 0
    stack := make([]int, 100)
    firstTwoCharacters := make([]byte, 0, 8)
    arrayStart := 0
    _ = eof

//line decoder.go:95
	{
	cs = ragelbug_start
	top = 0
	}

//line decoder.go:101
	{
	var _klen int
	var _trans int
	var _acts int
	var _nacts uint
	var _keys int
	if p == pe {
		goto _test_eof
	}
	if cs == 0 {
		goto _out
	}
_resume:
	_keys = int(_ragelbug_key_offsets[cs])
	_trans = int(_ragelbug_index_offsets[cs])

	_klen = int(_ragelbug_single_lengths[cs])
	if _klen > 0 {
		_lower := int(_keys)
		var _mid int
		_upper := int(_keys + _klen - 1)
		for {
			if _upper < _lower {
				break
			}

			_mid = _lower + ((_upper - _lower) >> 1)
			switch {
			case data[p] < _ragelbug_trans_keys[_mid]:
				_upper = _mid - 1
			case data[p] > _ragelbug_trans_keys[_mid]:
				_lower = _mid + 1
			default:
				_trans += int(_mid - int(_keys))
				goto _match
			}
		}
		_keys += _klen
		_trans += _klen
	}

	_klen = int(_ragelbug_range_lengths[cs])
	if _klen > 0 {
		_lower := int(_keys)
		var _mid int
		_upper := int(_keys + (_klen << 1) - 2)
		for {
			if _upper < _lower {
				break
			}

			_mid = _lower + (((_upper - _lower) >> 1) & ^1)
			switch {
			case data[p] < _ragelbug_trans_keys[_mid]:
				_upper = _mid - 2
			case data[p] > _ragelbug_trans_keys[_mid + 1]:
				_lower = _mid + 2
			default:
				_trans += int((_mid - int(_keys)) >> 1)
				goto _match
			}
		}
		_trans += _klen
	}

_match:
	_trans = int(_ragelbug_indicies[_trans])
	cs = int(_ragelbug_trans_targs[_trans])

	if _ragelbug_trans_actions[_trans] == 0 {
		goto _again
	}

	_acts = int(_ragelbug_trans_actions[_trans])
	_nacts = uint(_ragelbug_actions[_acts]); _acts++
	for ; _nacts > 0; _nacts-- {
		_acts++
		switch _ragelbug_actions[_acts-1] {
		case 0:
//line decoder.rl:18

        stack[top] = cs; top++; cs = 4; goto _again

//line decoder.go:186
		}
	}

_again:
	if cs == 0 {
		goto _out
	}
	p++
	if p != pe {
		goto _resume
	}
	_test_eof: {}
	if p == eof {
		__acts := _ragelbug_eof_actions[cs]
		__nacts := uint(_ragelbug_actions[__acts]); __acts++
		for ; __nacts > 0; __nacts-- {
			__acts++
			switch _ragelbug_actions[__acts-1] {
			case 1:
//line decoder.rl:22

        if err = callbacks.OnArrayData(data[arrayStart:p]); err != nil {
            p++; goto _out

        }
        if err = callbacks.OnArrayEnd(); err != nil {
            p++; goto _out

        }
        firstTwoCharacters = firstTwoCharacters[:0]
        top--; cs = stack[top]
goto _again

//line decoder.go:221
			}
		}
	}

	_out: {}
	}

//line decoder.rl:67

    if cs == ragelbug_error {
        err = fmt.Errorf("Parse error at %v", p)
    }

    return
}

parser_test.go

package ragelbug

import (
	"fmt"
	"testing"
)

type testCallbacks struct {
	Value            interface{}
	currentArray     []byte
	currentArrayType ArrayType
}

func (this *testCallbacks) OnArrayBegin(arrayType ArrayType) error {
	this.currentArray = make([]byte, 0, 100)
	this.currentArrayType = arrayType
	return nil
}

func (this *testCallbacks) OnArrayData(bytes []byte) error {
	this.currentArray = append(this.currentArray, bytes...)
	return nil
}

func (this *testCallbacks) OnArrayEnd() error {
	array := this.currentArray
	this.currentArray = nil
	switch this.currentArrayType {
	case ArrayTypeBinary:
		this.Value = array
	case ArrayTypeString:
		this.Value = string(array)
	default:
		return fmt.Errorf("Unhandled array type: %v", this.currentArrayType)
	}
	return nil
}

func decodeDocument(encoded []byte) (result interface{}, err error) {
	callbacks := new(testCallbacks)
	err = Parse(encoded, callbacks)
	if err != nil {
		return nil, err
	}
	result = callbacks.Value
	return result, err
}

func TestUnquotedString(t *testing.T) {
	expected := "abcd"
	actual, err := decodeDocument([]byte(expected))
	if err != nil {
		t.Errorf("Parse error: %v", err)
	}
	if actual != expected {
		t.Errorf("Expected [%v] but got [%v]", expected, actual)
	}
}

Build command (ragel 6.10, go 1.13.3):
go mod init ragelbug
ragel -Z decoder.rl && go test

Result:
--- FAIL: TestUnquotedString (0.00s)
panic: runtime error: index out of range [5] with length 4 [recovered]
	panic: runtime error: index out of range [5] with length 4

goroutine 19 [running]:
testing.tRunner.func1(0xc0000b2100)
	/usr/local/go/src/testing/testing.go:874 +0x3a3
panic(0x5333e0, 0xc0000b6000)
	/usr/local/go/src/runtime/panic.go:679 +0x1b2
ragel-bug.Parse(0xc00008c078, 0x4, 0x8, 0x56dce0, 0xc00007c2a0, 0x8, 0x4)
	decoder.go:129 +0x7d4
ragel-bug.decodeDocument(0xc00008c078, 0x4, 0x8, 0xc00008c078, 0x4, 0x8, 0xc1203b284f84)
	/home/karl/Projects/ragel-bug/parser_test.go:41 +0x6e
ragel-bug.TestUnquotedString(0xc0000b2100)
	/home/karl/Projects/ragel-bug/parser_test.go:51 +0x65
testing.tRunner(0xc0000b2100, 0x54f898)
	/usr/local/go/src/testing/testing.go:909 +0xc9
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:960 +0x350
exit status 2
FAIL	ragel-bug	0.005s

The failing line (decoder.go line 129, p = 5, data length = 4):
case data[p] < _ragelbug_trans_keys[_mid]:

Hi Karl, thank you for the detailed, minimal report!
This is caused by the fret in an EOF action, without backing up the input pointer. You can fix it with an fhold in front of the fret.
Interestingly, this IS Cloudbleed, golang edition. They turned on EOF actions by setting eof in a ragel program that hadn't changed for years and caused an fgoto that wasn't preceded by fret to execute on EOF. Gnarly stuff, I know. Anyhow, in ragel 6 it is fixed for C (284f1fb0ba7c2a5de6439761f0df0cc74f96e8b8), but not for Go.
We either need to port the fix to golang, or you can use ragel 7. It is fixed for all languages there, by virtue of the intermediate language.
As an aside, the intermediate language in version 7 is there for exactly this reason, we shouldn't have to port every fix and feature change to every language supported by ragel.

authors: ehdtee kstenerud

repo: ehdtee/colm-suite


text:
Handle CVE issues on got

Hi @ehmad11
There is some CVE on the got package.
Are you available to merge and publish if I suggest a PR to handle this issue ?
Thanks!

hi @julbrs,
Yes, sure, but the latest got  package itself is now native ESM and no longer provides a CommonJS export. So, ideally, we need to update this library too.

Hi @ehmad11!
Thanks for your quick feedback! I have worked a bit on netsuite-rest to migrate to ESM (guide suggested here), so it's possible:
https://github.com/ehmad11/netsuite-rest/compare/master...julbrs:netsuite-rest:esm
But then any project that is relying on netsuite-rest will break...
julien@LMON0015 utils % DEBUG=* node netsuite_import_test_data.js
/xx/netsuite_import_test_data.js:1
const NsApiWrapper = require('netsuite-rest')
                     ^

Error [ERR_REQUIRE_ESM]: require() of ES Module /xx/netsuite-rest.js from /xx/netsuite_import_test_data.js not supported.
Instead change the require of netsuite-rest.js in xx/netsuite_import_test_data.js to a dynamic import() which is available in all CommonJS modules.
    at Object.<anonymous> (/xxx/netsuite_import_test_data.js:1:22) {
  code: 'ERR_REQUIRE_ESM'
}

What is the best option here? Force ESM migration for all ? I am a newbie on CJS/ESM stuff 😅 thinking of starting a brand new TS project for that 🤓

I will now get a look at the suiteql package that is depending on this one,

@ehmad11 I can take some resp. / ownership on both projects if you don't have time / access to a Netsuite system to run tests. Let me know! I would like also to provide types (so probably migrate the projects to TS !)

@julbrs, just saw your package; it looks great. I am thinking, can we keep this current package as it is? Many projects depend on it, and switching to ESM would be a breaking change for them. We can put a deprecated notice in this package and advise users to use your package for future updates. I would really appreciate it if we could keep the new package as a fork of the current repo to preserve the commit history and contributions of other members with full credits.

and yes we can merge the suiteql, it is a great idea

That's a very good idea!
I have take rebuilt the new repository here: https://github.com/julbrs/netsuite-api-client which is a github fork of https://github.com/ehmad11/netsuite-rest ( because there is more contribution in this one).
There is just one more commit with all the new stuff (esm/typescript/merging).
I have also added a specific section in the README to explain the ESM/CommonJS stuff, with links to netsuite-rest and suiteql if user want to grab the CommonJS version on npmjs.
Thanks !

source: gharchive/issue

license: MIT

license_type: permissive

license_source: github-api


text:
Refactor simple_deploy.py

The Fly refactoring work in #269 is going well. It's pointing to some simple_deploy refactoring that I want to keep separate from that issue, because it will affect all platforms.

[ ] Work through this entire file, and make sure it reflects clear current thinking about the project.
[ ] Refactor rest of cli.py.
[ ] Consider moving some methods out of this class to a utils file of some sort.

[ ] Consider an sd_utils and a platform_utils.

[ ] Write unit tests for any critical units that are likely to be stable.
[ ] Write a clear module-level docstring. Discuss public methods, and how this module interacts with things like platform-specific Deployer classes. (Don't check this task until all refactoring work is done; make sure reflects final state of refactoring work.)
[ ] Methods like execute_subp_run() should log the commands that are run. That will remove a bunch of extraneous log lines from the platform-specific deploy modules.
[ ] Write a clear docstring about public methods, ie how this can be used by platform-specific deploy.py modules.
[ ] Run black against this file, and any related utils files.
[ ] Consider writing an ABC for the platform deployer classes. (That's not this file, but it will enforce the expectations that this file has for platform-specific code.)
[ ] Check for any necessary documentation updates.
[ ] Merge; probably no new release required.

Documentation update

[ ] Describe, very broadly, how this class works. What gets called, what does it do, and most importantly, how doe it interact with a PlatformDeployer class?
[ ] Consider moving some methods out of this class to a utils file of some sort.

Testing update
I've been avoiding starting unit tests because I called integration tests unit tests already, and I called e2e tests integration tests. I don't need to wait for renaming those to start actual unit tests.

[ ] Make a directory called real_unit_tests. When renaming other tests, I can easily rename this directory.
[ ] Write actual unit tests for utilities that are pulled out of the Command class.

Working notes

[ ] Update module-level docstring.
[ ] Remove unused imports.

All functions

[ ] __init__()
[ ] create_parser()
[ ] add_arguments()
[ ] handle()
[ ] _parse_cli_options()
[ ] _validate_command()
[ ] _validate_platform_arg()
[ ] _confirm_automate_all()
[ ] _start_logging()
[ ] _create_log_dir()
[ ] _ignore_sd_logs()
[ ] _strip_secret_key()
[ ] _inspect_system()
[ ] _inspect_project()
[ ] _find_git_dir()
[ ] _check_git_status()
[ ] _diff_output_clean()
[ ] _get_dep_man_approach()
[ ] _check_using_poetry()
[ ] _get_current_requirements()
[ ] _add_simple_deploy_req()
[ ] _get_req_txt_requirements()
[ ] _get_pipfile_requirements()
[ ] _write_pipfile_pkg()
[ ] _get_poetry_requirements()
[ ] write_output()
[ ] log_info()
[ ] execute_subp_run()
[ ] execute_command()
[ ] add_packages()
[ ] add_package()
[ ] _add_req_txt_pkg()
[ ] _add_poetry_pkg()
[ ] _check_poetry_deploy_group()
[ ] _add_pipenv_pkg()
[ ] get_confirmation()
[ ] commit_changes()

Writing output

[ ] Rework write_output() and log_info() to be logically consistent.
[ ] Move both of these to utils.
[ ] Make sure unit tests pass.
[ ] Run one integration test.
[ ] Write an actual unit test, or set of tests, for these functions.

_inspect_project()

[ ] self.project_name -> self.local_project_name?

Unit testing git status checks
(cont. from above)

[ ] Make each pass when run separately.
[ ] Sort test suite issues.
[ ] Update any other test failures.
[ ] Run integration tests.

Test suite issues
All tests pass in isolation, but suite fails. ie, test_sdlogs_exists() fails. Jumping to the temp dir shows this:
% cd ../../pytest-current/blog_project0
eric@Erics-Mac-Studio blog_project0 % git status
On branch main
Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
	modified:   .gitignore
	modified:   blog/settings.py
	modified:   blog/wsgi.py
	modified:   requirements.txt

Untracked files:
  (use "git add <file>..." to include in what will be committed)
	.dockerignore
	Dockerfile
	fly.toml

no changes added to commit (use "git add" and/or "git commit -a")

That's a bunch of cruft from previous tests that passed:
platform_agnostic_tests/test_git_status_checks.py::test_clean_git_status[req_txt] PASSED                    [  3%]
platform_agnostic_tests/test_git_status_checks.py::test_unacceptable_settings_change[req_txt] PASSED        [  6%]
platform_agnostic_tests/test_git_status_checks.py::test_unacceptable_file_changed[req_txt] PASSED           [ 10%]
platform_agnostic_tests/test_git_status_checks.py::test_sdlogs_exists[req_txt] FAILED                       [ 13%]

[ ] Make sure temp project is reset after each test run. Look at how this is done in other platform-agnostic tests.

More test suite notes

[ ] Move reset_test_project_function() to test_git_status_checks.py module, and set autouse=True. (Good, it's being called every time, but still not letting all tests pass.)
[ ] Pass test_unacceptable_settings_change individually.
[ ] Pass test_unacceptable_settings_change as part of suite.

New tests

[ ] Test that simple_deploy_logs/ is added to .gitignore?

Git status checks
From a unit test run, this output is failing the git status check:
% git status --porcelain
 M .gitignore
(b_env) eric@Erics-MacBook-Air blog_project0 % git diff --unified=0
diff --git a/.gitignore b/.gitignore
index 9c96d1b..4279ffb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,0 +9,3 @@ db.sqlite3
+
+# Ignore logs from simple_deploy.
+simple_deploy_logs/

Move call to prep_automate_all()
Right now, prep_automate_all() is called from simple_deploy. The only thing after it is adding simple_deploy as a requirement. This is a bunch of complexity with little benefit. The only benefit was not adding simple_deploy as a requirement until after prepping automate-all. But we've already added a log file. People can use Git to undo changes if they decide to abandon simple_deploy.

[ ] Call prep_automate_all() from simple_deploy's handle() to PlatformDeployer.deploy().
[ ] Call validate_platform() from deploy() as well.
[ ] Run all local tests.
[ ] Run integration tests.

Final handle() cleanup

[ ] Can platform_deployer() be created later, now that most platform-specific work has been moved out of simple_deploy.py?

_add_poetry_pkg()

[ ] Looks like check for poetry deploy group will fail if there's any slight formatting differences.

[ ] Write test for this?
[ ] Update code to more reliably check for a pre-existing deploy group?

_add_poetry_pkg()

[ ] Looks like check for poetry deploy group will fail if there's any slight formatting differences.

[ ] Write test for this?
[ ] Update code to more reliably check for a pre-existing deploy group?

Misc cleanup

[ ] Convert some util functions to helpers.

Parking lot: git status checks
I saw one issue around newlines, with a diff that included a + and - of the same line. I'm not sure if that would be a common thing, or if that was a weird one-off issue. Keep this out for now, but keep it here in case it comes up again.
def test_clean_diff():
    diff_output = dedent(
        """\
        diff --git a/blog/settings.py b/blog/settings.py
        index 6d40136..6395c5a 100644
        --- a/blog/settings.py
        +++ b/blog/settings.py
        @@ -39,0 +40 @@ INSTALLED_APPS = [
        +    'simple_deploy',
        @@ -134 +135 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
        -LOGIN_URL = 'users:login'
        \\ No newline at end of file
        +LOGIN_URL = 'users:login'"""
    )

    cleaned_diff = sd_utils._clean_diff(diff_output.splitlines())
    assert cleaned_diff == ["+    'simple_deploy',"]

    # Ignore changes that relate to newlines, like this:
    #     -LOGIN_URL = 'users:login'            <-- this line
    #     \\ No newline at end of file
    #     +LOGIN_URL = 'users:login'"""         <--- and this line
    # Remove the + or -, and find any lines that are repeated.
    changes = [l[1:] for l in lines]
    changes_to_remove = [c for c in changes if changes.count(c) == 2]
    lines = [l for l in lines if l[1:] not in changes_to_remove]

Move remaining platform-specific work to platform-specific module
The only platform-specific work that's still being done in simple_deploy is confirming the user wants to use a platform that only has preliminary support. But that work can be done after platform-agnostic work is done. This means all platform-specific work is done entirely in the platform-specific deployer module.

[ ] Move instantiation of platofrm_deployer object to end of handle().
[ ] Call confirm_preliminary() from platform-specific deployer modules.
[ ] Run unit tests.
[ ] Run integration tests.

[ ] fly_io
[ ] fly_io automate_all
[ ] platform_sh
[ ] heroku

fly automate_all test

[ ] Sort out _confirm_use_attached_db() arg issue. Should it return True during automate-all runs?

Manual run: fly automate_all

[ ] Confirmed fly, but didn't confirm automate all until after creating app and db! Fix this.

repo: ehmatthes/django-simple-deploy


text:
Add a variable to control the row height for header

Use a specific row height that is varied for both mobile and desktop that will then be useable as a variable going forward in the onepage theme as well.

Completed.

repo: ehowey/gatsby-theme-catalyst-core


text:
Height of bars sometimes not correct

I am seeing an odd behavior that sometimes the height of bars is not correct.
Here's an example...
In a seven day statistic every bar has the right height:

Now, when I only output the data of the current month (five days) the bars are too high - to be clearly seen at the first green bars, which should be 4 and 5:

The underlying data for the days is exactly the same for both charts.
Have I checked? Yes I did:

Any ideas?
Thanks a lot.

I think the problem is, that when the bar data changed, the indicator values are not being refreshed. With the second chart, the upper indicator value should be 7, not 8.

Hi @stephansann , yeah as i tested, it seems indicators are not refreshing depends on data.
I will try to fix it as soon and publish new version.

@stephansann , new version released.
test 0.0.16 version and let me know if it fixed.

Your welcome, thank you for reporting the problem.

authors: ehsannarmani stephansann

repo: ehsannarmani/ComposeCharts


text:
Option request: Auto Store only in containers that already have 1+ of the item

Is there an existing issue for this?

[X] I have searched the existing issues

Mod version
1.4.3
Is this a bug caused by this mod alone?
The bug was reproduced with this mod alone
List of mods to reproduce bug
No response
What happened?
Not a bug, obviously, but a feature request. I'd like a toggle that would only allow stations to auto store in containers that already contain at least one of the item. If none are found, drop the output on the ground.
That would really help organize the output and reduce the randomness. For example, right now I have a couple of smelters outputting to multiple chests nearby. I just want them all to go into a single chest. I realize I can reduce the radius for pickup and not have as many chests near, but many other "auto store" mods have this feature and it's very handy and something I've come to rely on.
Steps to reproduce the bug

Have multiple machines producing items within range of multiple chests.
Items go into multiple chests rather than a single chest.

Logs and screenshots
No response

I will add this option to the TODO list.
Sorry, but since this issue is not a bug report, I will close it.

authors: Darrken eideehi

repo: eideehi/valheim-automatics


text:
request metrics

Starts the already implemented metric exporter in cloudrunner.Run, and starts collecting metric from gRPC servers and clients initialized via this package.
Waiting for a release including https://github.com/open-telemetry/opentelemetry-go-contrib/pull/1546 being cut.

This is great! It would be even better if we got this in upstream in the contrib repo as gRPC-Go metrics addon - otherwise we'll need to migrate to new metric names once that gets released.

Opened dialog with OTEL maintainers regarding this. I still think we should move forward here.

source: gharchive/pull-request

repo: einride/cloudrunner-go


text:
Add development instructions

These are the steps @hobozhang and I did to get this set up locally.

This is great! Other than that one comment, 👍

authors: Nosajool ejholmes

repo: ejholmes/slashdeploy


text:
Xml builder

This PR swaps the mechanics of building an xml file with a library dedicated to it. This means we don't have to worry about not properly escaping xml entities anymore.
The PR is 100% compatible with the existing sitemap API and is as performant. Aside from whitespace changes it generates the same xml too.
see #135 for performance comparison
make test-perf runs=100
node tests/perf.js 100
runs: 100
=========  sitemap creation  =============
mean: 3.4
median: 2.9
variance: 2.2
standard deviation: 1.5
90th percentile: 5.8
99th percentile: 9.1
=========  sync  =============
mean: 669.7
median: 667.7
variance: 7908.7
standard deviation: 88.9
90th percentile: 815.2
99th percentile: 854.4
=========  async  =============
mean: 664.3
median: 666.2
variance: 4297.0
standard deviation: 65.6
90th percentile: 769.9
99th percentile: 827.3

@realityking @ekalinin I would like your opinions as to whether switching over to building this xml via the library is a good move for the project. Please let me know if you have any objections.
I included the performance test to address a possible concern there.

Hey @derduher in general LGTM.
But could you return back semicolons (at least in lib/sitemap.js)?
In this case the diff will be looking much more clearer.

@ekalinin sorry about that. I had my formatter on (standardjs). I've added back semicolins to all the lines where they were the only difference. Here is a direct link to the diff with whitespace diffs collapsed.

sorry about that. I had my formatter on (standardjs). I've added back semicolins to all the lines where they were the only difference.

No problems. Thanks for the patch!

authors: derduher ekalinin

license: mit

license_source: bigquery


text:
Record a code editing

For review interviewing require to record the interview. Maybe add this functional to this project?

Rustpad does not support any recording features.

Ok

authors: daxartio ekzhang

repo: ekzhang/rustpad


text:
Can't find gdlint

The problem happens with 2.0.0 from the AssetLib but also with current master.
Version 1.0.1 still works.
I'm using Godot 4.3.dev5.

Thank you for reporting, it's a known issue and someone has also a fix for it in https://github.com/el-falso/gdlinter/issues/4#issuecomment-2011405458. I'll push out the fix when I'm on the PC again.

I'm on linux and still get the error with 2.0.1. I'll see if I can find out where the problem is.

So the problem seems to be in get_gdlint_path.
In my case it returns ~/.local/bin/gdlint, but it's not there, it's at /usr/bin/gdlint in my case. Returning just "gdlint" as for windows works for me.

Thank you for reproducing the issue. Seems like it's more difficult to get the path to gdlint for various operating systems and for various different installations than I thought.
Hope #10 fixes it this time, since I added two fallbacks.

:+1: it's working now.
Thanks for creating this awesome plugin!

Great, thank you really much for your kind words. ❤
I really hope this issue will be fixed forever now, since I'm really out of solutions. 😅

authors: el-falso eldyer

repo: el-falso/gdlinter


text:
Added test html file

Hey Elad, I really love your work!
I published a css reset with similar goals a while back (https://www.npmjs.com/package/reset-css-complete) but I dig your approach so much more!
Anyway, I thought I could share the simple html file I used for testing across browsers and thought that it might help you as well (for example, did you know that in chrome unsetting <input type="color"> is totally broken?)
Hope you find this useful
-Alex

It isn't a bug. It removes the basic style of the "button".
If you add size like 'width: 100px; height: 100px;' you will see your custom styles.

I'm not sure whether you checked chrome or not but on my latest Version 91.0.4472.164 (Official Build) (x86_64) the color input is a 6 x 818px line when you unset the properties. If the below doesn't look like a UI bug I don't know what to say 😄

In comparison firefox resets the color input properly to a 1px x 1px square.
Obviously it's not a bug with this reset file (and lets be honest, who even uses color inputs), but it seems to be a bug in chrome and if you add your own styles on top it's easy to fix. I thought it was an interesting find though - and the reason I volunteered to contribute my test file!

There are two separate issues here:

There should be a simple HTML page with a preview of all browser tags, so one can preview it on the website without having to install/import the package.
If some of the styling is broken, it should be a separate issue. @alex-e-leon can you please open a new issue with a description and a codepen/codesandbox demonstrating the problem?

@elektronik2k5 thanks for adding some clarity - I agree.
My intention with this PR was to address #1.
I went ahead and looked into #2 anyway to see if I could get to the bottom of it, and it turns out that the issue was caused by my testfile not including the proper Doctype header, so it looks like it's only an issue in chrome's quirks mode (I still find it to be a strange outcome though!)
I've updated my PR to include the html5 doctype and unset now behaves as expected in chrome, safari and firefox when I tested it.

@alex-e-leon I tested it in Chrome 91 and 92; it looks fine and not spreading.
Of course, to see it well, it needs to get sizing, but this is the idea of this CSS reset.
CodePen Demo Reset Color Input: https://codepen.io/elad2412/pen/83c4595573dd6ff280533e2b17fb7ff6?editors=1100
For the test, it will wait a little bit. I'm thinking of adding another CSS that will change the basic style with examples for form HTML elements on how to design them.
And after that, I will add your test.
Thanks,
Elad


text:
Add speech-to-text workflow

Workflow allows failures, i.e. schedule-and-upload won't fail if speech-to-text does (which happens when there is no audio or speech in a video).

I got no idea about workflows, lets see if this works :grimacing:

repo: elan-ev/opencast-tobira


text:
kafka: manager: List topics before creating them

Updates the manager CreateTopics method to only issue the CreateTopics call if the topics don't exist in Kafka, preventing undesirable errors from being returned and logged.

I've opened https://github.com/elastic/apm-queue/issues/318 to investigate.

repo: elastic/apm-queue


text:
EcsTextFormatter use of BaseStream results in unbuffered writes which causes out of order writes to file in Serilog

When writing the formatted json to a Serilog Durable Http Sink (Serilog.Sinks.Http), the log is corrupted as a BOM is written after the first log rather than as the first bytes of the file.  The Http Sink in turn uses a file sink to store the buffer before shipping to http endpoint (Logstash in my case).  Logstash cannot process the logs due to an embedded BOM.
The cause of the issue comes down to the EcsTextFormatter writing directly to the base stream rather than the buffer.  When the Serilog File Sink flushes the buffer for the first time, the BOM is written after the first log rather than being the first bytes.
https://github.com/elastic/ecs-dotnet/blob/d64e2fd57c8dd3b4ee193d283aee7ef5a2460673/src/Elastic.CommonSchema.Serilog/EcsTextFormatter.cs#L29
Please see Issue raised on Serilog project where this was discussed, prior to coming to conclusion that EcsTextFormatter should be modified.
To prove this, changing the code to below results in the correct sequence of writes:
output.Write(ecsEvent.Serialize());

Thanks for opening @Jhaiii, looking at now

authors: Jhaiii russcam

repo: elastic/ecs-dotnet


text:
Adding Nodes in ES-Yarn

Is there is a simple way to scale up more nodes in ES Yarn?

es-yarn allows you to specify the number of nodes at startup. Changing the
number of required resources cannot be done dynamically in YARN (currently)
so one would have to either restart the process (that is, stop the current
setup and create a new one with the new settings) or potentially start a
different es-yarn configuration with the delta resources; if the same setup
is used, the new nodes should be able to pick the old ones up.
However, as I've mentioned this goes beyond the official features in YARN
and thus, it is quite brittle and might not work at all.
Cheers,
P.S. This is an issue tracker, not a forum - please use the mailing list
next time for asking questions. Thanks!
On Thu, Apr 30, 2015 at 9:57 PM, WalterDalton notifications@github.com
wrote:

Is there is a simple way to scale up more nodes in ES Yarn?
—
Reply to this email directly or view it on GitHub
https://github.com/elastic/elasticsearch-hadoop/issues/443.

authors: WalterDalton costin

repo: elastic/elasticsearch-hadoop


text:
encodes : and | when using them in url

Thank you for submitting a pull request!
Please make sure you have signed our Contributor License Agreement (CLA).
We are not asking you to assign copyright to us, but to give us the right to distribute your code without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code.
You only need to sign the CLA once.

[ X] I have signed the Contributor License Agreement (CLA)

I think the CLA is just a little slow on the uptake. Hopefully it will refresh.

Looks like it hasn't updated yet. I'll see what's going on.

I don't think this should be merged. Please see comment on #874

@megri The issue that this PR is solving is a org.apache.commons.httpclient.URIException: Invalid query, which is thrown during the URI parsing before the request is even sent to Elasticsearch. I am thinking this has to do with an unsupported version of commons httpclient. That being said, I think that this is still a win if it means that a simple encoding change will allow a different client version to just work, even if it's not officially supported.

Hi @bwf93 , we have your CLA signature on file, but with a different e-mail. Can you please add both to your Github profile (they can be hidden) so we can correctly match your signature to your Github account and the Git commit?

Due to time constraints surrounding backwards compatibility testing, I elected to implement a fix for #874 in commit https://github.com/elastic/elasticsearch-hadoop/commit/d442bb4479c2ae8478a10f25a7f0edb6b7256d87. The commit should catch the issues referenced in this PR. Because of this, the PR is no longer needed, and will be closed.

authors: bwf93 jbaiera karmi megri


text:
Index concrete types and search them by common base type easily

I know, this is a demanding feature request. Who dares wins ;-)
Assume the following models...
abstract class MediaPoco { public int Id { get; set; } }
class ImagePoco : MediaPoco { ... }
class DocumentPoco : MediaPoco { ... }

Now, I'd like to get all images and documents:
var settings = new ConnectionSettings(uri, "all_media");
settings.MapDefaultTypeNames(desc => desc
    .Add(typeof (ImagePoco), "image")
    .Add(typeof (DocumentPoco), "document")
    );
var client = new ElasticClient(settings);
var allMedia = client.Search<MediaPoco>(desc => desc.AllTypes());

Of course I get a JsonSerializationException: Could not create an instance of type MediaPoco. Type is an interface or abstract class and cannot be instantiated.
The obvious approach is providing a ConcreteTypeSelector for every search request:
var allMedia = client.Search<MediaPoco>(desc => desc
    .AllTypes()
    .ConcreteTypeSelector((o, hit) => hit.Type == "image" ? typeof (ImagePoco) : typeof (DocumentPoco)));

My kind request would be:
Since you know the concrete type mapping from beginning either by MapDefaultTypeNames(..) or ElasticTypeAttribute could you deserialize with that types and not with the T in client.Search<T>(..)?

If you use .Types(typeof(ImagePoco), typeof(DocumentPoco)) we'll build the concrete type selector for you. Does that work for you?

That's more helpful! Thanks
But finally I'm looking something like that
var allMedia = client.Search<MediaPoco>(desc => desc
    .Types(typeof (MediaPoco).Assembly
        .GetTypes()
        .Where(type => type.IsSubclassOf(typeof (MediaPoco)))
        .ToArray())
    );

Yeah, that will work. Just in case I would like a VideoPoco.

Cool! Just to be clear that behaviour on Types() already exist now so closing this issue :+1:
The covariance section on our search basics docs covers this as well. http://nest.azurewebsites.net/nest/search/basics.html

repo: elastic/elasticsearch-net


text:
Provide Xmx configuration as an option from command line

A large cluster state can cause the tool to go oom.
Exception in thread "main" java.lang.OutOfMemoryError: Java heap space

at java.util.Arrays.copyOf(Unknown Source)

at java.lang.AbstractStringBuilder.expandCapacity(Unknown Source)

at java.lang.AbstractStringBuilder.ensureCapacityInternal(Unknown Source)

at java.lang.AbstractStringBuilder.append(Unknown Source)

at java.lang.StringBuilder.append(Unknown Source)

at org.springframework.util.StreamUtils.copyToString(StreamUtils.java:74)

at org.springframework.http.converter.StringHttpMessageConverter.readInternal(StringHttpMessageConverter.java:88)

at org.springframework.http.converter.StringHttpMessageConverter.readInternal(StringHttpMessageConverter.java:40)

at org.springframework.http.converter.AbstractHttpMessageConverter.read(AbstractHttpMessageConverter.java:159)

at org.springframework.web.client.HttpMessageConverterExtractor.extractData(HttpMessageConverterExtractor.java:106)

at org.springframework.web.client.RestTemplate$ResponseEntityResponseExtractor.extractData(RestTemplate.java:809)

at org.springframework.web.client.RestTemplate$ResponseEntityResponseExtractor.extractData(RestTemplate.java:793)

at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:572)

at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:530)

at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:448)

at com.elastic.support.diagnostics.RestModule.submitRequest(RestModule.java:27)

at com.elastic.support.diagnostics.commands.RunClusterQueriesCmd.execute(RunClusterQueriesCmd.java:39)

at com.elastic.support.diagnostics.DiagnosticChain.execute(DiagnosticChain.java:31)

at com.elastic.support.diagnostics.DiagnosticApp.main(DiagnosticApp.java:47)

This stacktrace is not coming from elasticsearch.
Did you report in the wrong place?
BTW we do support setting memory options.

Oops. Sorry.

With the revised call mechanism this should be less of an issue.  You can bump up the -Xmx with the DIAG_JAVA_OPTS environment variable similar to how you would do in ES so I'm closing this one.

authors: dadoonet nemonster sherry-ger


text:
test: use context cancel in appender benchmarks for more accurate results

We can't pass context.Background() to the indexer because it's using a nil channel under the hood, speeding up the select statement. That creates misleading benchmark results as they do not reflect production usage.

LGTM. Did you happen to run benchstat? If so, could you share the results?

                                      │  before.txt  │              after.txt               │
                                      │    sec/op    │    sec/op     vs base                │
Appender/NoCompression-20               506.7n ±  6%   514.0n ±  6%        ~ (p=0.123 n=10)
Appender/NoCompressionScaling-20        867.6n ± 13%   872.5n ± 41%        ~ (p=1.000 n=10)
Appender/BestSpeed-20                   1.212µ ± 13%   1.408µ ±  4%  +16.13% (p=0.001 n=10)
Appender/BestSpeedScaling-20            1.325µ ±  9%   1.391µ ±  9%   +4.94% (p=0.043 n=10)
Appender/DefaultCompression-20          1.757µ ±  1%   1.806µ ±  4%   +2.82% (p=0.000 n=10)
Appender/DefaultCompressionScaling-20   1.751µ ±  1%   1.810µ ±  3%   +3.37% (p=0.000 n=10)
Appender/BestCompression-20             1.742µ ±  2%   1.804µ ±  2%   +3.59% (p=0.000 n=10)
Appender/BestCompressionScaling-20      1.753µ ±  2%   1.799µ ±  1%   +2.65% (p=0.000 n=10)
geomean                                 1.267µ         1.322µ         +4.34%

                                      │   before.txt   │               after.txt               │
                                      │      B/s       │      B/s       vs base                │
Appender/NoCompression-20                252.2Mi ±  6%   248.6Mi ±  6%        ~ (p=0.123 n=10)
Appender/NoCompressionScaling-20         147.3Mi ± 11%   146.5Mi ± 70%        ~ (p=1.000 n=10)
Appender/BestSpeed-20                   105.46Mi ± 11%   90.80Mi ±  4%  -13.90% (p=0.001 n=10)
Appender/BestSpeedScaling-20             96.46Mi ± 10%   91.89Mi ±  8%   -4.75% (p=0.043 n=10)
Appender/DefaultCompression-20           72.76Mi ±  1%   70.76Mi ±  4%   -2.74% (p=0.000 n=10)
Appender/DefaultCompressionScaling-20    73.00Mi ±  1%   70.62Mi ±  3%   -3.27% (p=0.000 n=10)
Appender/BestCompression-20              73.38Mi ±  2%   70.84Mi ±  2%   -3.46% (p=0.000 n=10)
Appender/BestCompressionScaling-20       72.92Mi ±  2%   71.03Mi ±  1%   -2.59% (p=0.000 n=10)
geomean                                  100.9Mi         96.69Mi         -4.17%

                                      │ before.txt │             after.txt             │
                                      │    B/op    │    B/op     vs base               │
Appender/NoCompression-20               514.0 ± 0%   515.5 ± 0%  +0.29% (p=0.008 n=10)
Appender/NoCompressionScaling-20        533.0 ± 1%   515.0 ± 3%  -3.38% (p=0.002 n=10)
Appender/BestSpeed-20                   981.0 ± 0%   985.0 ± 1%       ~ (p=0.212 n=10)
Appender/BestSpeedScaling-20            984.5 ± 0%   983.5 ± 0%       ~ (p=0.925 n=10)
Appender/DefaultCompression-20          981.0 ± 3%   965.0 ± 4%       ~ (p=0.897 n=10)
Appender/DefaultCompressionScaling-20   984.0 ± 1%   949.0 ± 4%       ~ (p=0.109 n=10)
Appender/BestCompression-20             979.0 ± 5%   965.5 ± 4%       ~ (p=0.957 n=10)
Appender/BestCompressionScaling-20      980.5 ± 3%   944.5 ± 5%       ~ (p=0.239 n=10)
geomean                                 838.8        825.1       -1.64%

                                      │ before.txt │              after.txt              │
                                      │ allocs/op  │ allocs/op   vs base                 │
Appender/NoCompression-20               7.000 ± 0%   7.000 ± 0%       ~ (p=1.000 n=10) ¹
Appender/NoCompressionScaling-20        7.000 ± 0%   7.000 ± 0%       ~ (p=1.000 n=10) ¹
Appender/BestSpeed-20                   10.00 ± 0%   10.00 ± 0%       ~ (p=1.000 n=10) ¹
Appender/BestSpeedScaling-20            10.00 ± 0%   10.00 ± 0%       ~ (p=1.000 n=10) ¹
Appender/DefaultCompression-20          10.00 ± 0%   10.00 ± 0%       ~ (p=1.000 n=10) ¹
Appender/DefaultCompressionScaling-20   10.00 ± 0%   10.00 ± 0%       ~ (p=1.000 n=10) ¹
Appender/BestCompression-20             10.00 ± 0%   10.00 ± 0%       ~ (p=1.000 n=10) ¹
Appender/BestCompressionScaling-20      10.00 ± 0%   10.00 ± 0%       ~ (p=1.000 n=10) ¹

repo: elastic/go-docappender


text:
estransport: honour request context cancellation

Abort retrying when the request context has been cancelled.
Fixes https://github.com/elastic/go-elasticsearch/issues/300

LGTM!

Thanks @Anaethelion. Will you take care of backporting this to 7.x?

@axw Will do today !

authors: Anaethelion axw

repo: elastic/go-elasticsearch


text:
Getting Stateful Set error while deploying elasticsearch in EKS cluster

Chart version:
Latest
Kubernetes version:
1.19
Kubernetes provider: E.g. GKE (Google Kubernetes Engine)
EKS
Helm Version:
v3
helm get release output
e.g. helm get elasticsearch (replace elasticsearch with the name of your helm release)
Be careful to obfuscate every secrets (credentials, token, public IP, ...) that could be visible in the output before copy-pasting.
If you find some secrets in plain text in helm get release output you should use Kubernetes Secrets to managed them is a secure way (see Security Example).

Output of helm get release

Error: template: elasticsearch/templates/statefulset.yaml:298:27: executing "elasticsearch/templates/statefulset.yaml" at <.Values.roles.master>: can't evaluate field master in type interface {}

Describe the bug:
Steps to reproduce:

Expected behavior:
Provide logs and/or server output (if relevant):
Be careful to obfuscate every secrets (credentials, token, public IP, ...) that could be visible in the output before copy-pasting

Any additional context:

when I try to install helm install elasticsearch using below commands, receiving error
helm repo add elastic https://helm.elastic.co
curl -O https://raw.githubusercontent.com/elastic/helm-charts/master/elasticsearch/values.yaml
helm install --name elasticsearch elastic/elasticsearch -f ./values.yaml
Error:
Error: template: elasticsearch/templates/statefulset.yaml:298:27: executing "elasticsearch/templates/statefulset.yaml" at <.Values.roles.master>: can't evaluate field master in type interface {}

I am also facing this issue but I think this has been fixed in 8.0.0-snapshot?
https://github.com/elastic/helm-charts/commit/0626843a31bacc7954561b26cca7b413e951cb9f
So the only question is, when will the 8.0.0 chart be published.

I am also facing this issue but I think this is due to the breaking change in 8.0.0-snapshot?
0626843
So the only question is, when will the 8.0.0 chart be published.

Hi @knyuwork, 8.0.0-SNAPSHOT is for development only tracking the https://github.com/elastic/elasticsearch master branch for futur 8.0.0 release.
8.0.0 chart will be released only when Elasticsearch 8.0.0 will be released.

curl -O https://raw.githubusercontent.com/elastic/helm-charts/master/elasticsearch/values.yaml
helm install --name elasticsearch elastic/elasticsearch -f ./values.yaml

Elasticsearch values from master branches are only compatible with the future 8.0.0 which isn't released yet as mentioned in the previous comment.
When you are running helm install elasticsearch elastic/elasticsearch command, you are deploying the last released version (currently 7.13.2).
Note that with Helm, when you're deploying the chart all the default values from the values.yaml file are already deployed without adding -f ./values.yaml. You should use create your own values file and use -f ./values.yaml only to override some specific values.

OK, but how we install version 8.0.0 for tests only?

authors: Dhanabalan-pop botzill jmlrt knyuwork

repo: elastic/helm-charts


text:
Logstash 2.0 support

Is this project going to be updated to support logstash 2.0?
Looking back through the other issues this seems to be asked lots and there seems to be no answer forth coming from elastic. Can we get a concrete timeframe for some progress for this project?

+1

+1

:+1:

In the meantime, logstash 2.1 is released yesterday

we really need this, we utilize logstash 2.1 for some features only present in that version.

+1

+1

+1

+1

+1

+1

@electrical I know you were waiting til 2.0 release of all the changes to es module to port back here. Any update on the status of this?

To rest of thread what issues are you facing? I've installed 2.0 and 2.1 on CentOS with this module just fine.

Hi @cdenneen
I've deployed logstash 2 with this module and then tried to send output from filebeat. Logstash needed the filebeat plugin to be installed and I could not do that with the current module.
I've created my own module to download the logstash tarball which includes all the plugins I need.
Thanks
shaun

Here is basically what I did... I created a profile::logstash which includes the logstash module here.
I then added this define type:
define profile::logstash::plugin_install {
  if $name =~ /\.\d$/ {
    $plugininfo = split($name, '-')
    $pluginname = "${plugininfo[0]}-${plugininfo[1]}-${plugininfo[2]}"
    $pluginver = $plugininfo[3]
    $command = "/opt/logstash/bin/plugin install --version ${pluginver} ${pluginname}"
  }else{
    $command = "/opt/logstash/bin/plugin install ${name}"
  }
  exec {"installing ${name}":
    path    => ['/bin', '/usr/bin'],
    command => $command,
    unless  => "test -d /opt/logstash/vendor/bundle/jruby/1.9/gems/${name}*",
    require => Class[::logstash]
  }
}

This defined type allowed me to install modules as normal and even specify particular versions.
Then inside my profile::logstash I've included:
profile::logstash::plugin_install { ['logstash-codec-cloudtrail', 'logstash-codec-plain', 'logstash-filter-prune', 'logstash-filter-translate', 'logstash-input-sqs-1.1.0', 'logstash-input-lumberjack-1.0.5', 'logstash-filter-mutate-1.0.2']: }

While this probably isn't the best solution because it hardcodes paths and jruby version... it was a useable solution until @electrical ports all the ES changes back into this module.
Hope this helps

I can also install logstash 2.2.0 with the module.
I had the following output with logstash version 1.5.6:
output {
  elasticsearch {
    host => "localhost"
    port => 9200
    protocol => "http"
    manage_template => false
  }
}

and had to change it to
output {
  elasticsearch {
    hosts => "localhost:9200"
    manage_template => false
  }
}

:+1:

@electrical   Any chance you could give people an update on approximately when we are likely to have this module updated properly?

+100

@tylerjl  Is there a new maintainer here, or is there likely to be one?  I am writing a series of blog posts on how to build a scalable logstash solution with these and other modules.  It'd be so great if I can just say "hey, the logstash module just works!"

@alexharv074 it's on our radar - it'll have a place somewhere, whether that's under some of the infrastructure I'm overhauling right now or otherwise.

@tylerjl If I send in PRs will they be merged?

@alexharv074 I'd like to get CI stable again so we can merge confidently, but once that's done, given the changes are sane, I certainly don't see why not. :+1:

@tylerjl what's the issue with the CI?  I just ran all the tests and they're fine.  There's a Travis config in place.  All looks ok?

@cdenneen Is it your understanding that the only feature that's missing for the latest Logstash is the plugin installation?

@alexharv074 yes this is one of the more important parts to get the basic functionality of 1.5+. But as other open issues have mentioned it should support templates and multi-instance support.

@alexharv074 I'm referring to the entire suite of tests, which include acceptance tests, which as far as I know are not running anywhere cleanly right now, including on Jenkins (where there are no executors) and Travis (where this repo is disabled and hasn't run for several couple months.)

@tylerjl  Can you guide me on what I'd need to do to get the acceptance tests working again?

@alexharv074 I'm wrapping up overhauling testing in the puppet-elasticsearch repo, which I'll then carry over into this repo and enable everyone to run the tests as long as they've got make, bundler, and docker available.

This sounds like a lot of work.  I doubt that many people would agree that overhauling the automated testing should take priority over basic functionality.  Test driven development is great, but right now, we don't have a working product to test, and we haven't had one since Logstash 1.5 was released about one year ago.

@tylerjl   I have written the Elasticsearch chapter of my blog post series here.  I'm going to write another article discussing the base class and the Kibana 4 node.  I need this module to work, or I'm just going to have to fork it, fix it, and tell people to use my version.

@alexharv074 totally understand the timeline you're working under. If I can't get caught up to this module in time, we can always PR your downstream changes into this module later so you aren't blocked.

Well it seems you're doggedly resisting my efforts to be the squeaky wheel!  Fair enough.  The bigger issue is I feel this makes both Puppet and Elastic.co look bad.  I can't for the life of me understand why so much effort would have gone into the Elasticsearch Puppet module apparently at the expense of this one.  Anyhow, I realise you're new to this role so I'll leave you alone now.  Thanks for all your efforts.

Thanks @jarpy! I really hope this module will get it's much needed update and your test rework is one step in the right direction.

The acceptance tests are now running well in Travis, and the modern (>= 1.5.0) plugin system is now supported by the module.
I'll try now to define what complete 2.0 support looks like, so we can get everyone rolling.
Specific requests are very welcome at this stage. Thanks.

All existing functionality is now working and tested with Logstash 2.x, including the new plugin support. I'm going to declare the 2.0 work as "minimally done", and close this issue.
I'm certainly not saying that there's nothing left to do! But if you all could help me by creating individual issues for bugs and features, I'll be able to clearly see what the hot topics are, and hopefully deliver faster for you.
Let me know what's next! :dart:
Thanks.

authors: FireDrunk MikeMcMahon SanderSander alexharv074 beeva-diegogarcia bhaku brhahlen cdenneen jarpy lbischof ltutar mkrakowitzer paul-cs rabbenhuis shaunrampersad topperge tylerjl

repo: elastic/puppet-logstash


text:
Avoid redefining arguments

With this commit we reenable the pylint check R1704 which checks
whether arguments have been redefined as local variables.
Relates #838

Thanks for the quick review. :)

repo: elastic/rally


text:
[DISCUSS] acceptance: Tests fail with errors unrelated to TF provider

Description
Currently acceptance tests can sometimes fail when there are errors
unrelated to the terraform provider.
For example, if a deployment is unhealthy (unassigned shard or whatever),
or the deployment failed to delete (which only issues a warning), etc, the tests will fail.
It seems to me that it would probably make sense to either log these errors somehow,
or not take these errors into account at all as they can provide false positives.

What do you think @marclop ?

It would make sense to log something, but the deployment gets destroyed at the end of the test which doesn't make for much of a debugging workflow.
I've only seen the error show up when "plan failed due to unknown error" appears in the plan tracker logs.
* deployment [d2d9afb696d4b79baa642739e142c14a] - [kibana][046e06f37d8d4c44908dc9ba1d3c1ef3]: caught error: "plan failed due to unknown error"

This comes from the SDK:
https://github.com/elastic/cloud-sdk-go/blob/master/pkg/plan/track_step.go#L78-L86

The tests also seem to fail when the deployment is unhealthy 🤔 I'm not really sure we need to check deployment health?
There was also a time when the test reported a deployment had failed to destroy but when I went to check it had been a false positive.
I think it could be a good idea to relax checks a bit? idk, what do you think?

I think it's probably OK to remove the Healthy check

authors: karencfv marclop

repo: elastic/terraform-provider-ec


text:
同样源码,在centos7.4上打包无法注册到注册中心

代码使用spring boot启动, 使用注解bean形式初始化注册中心bean, 手动调用了init方法, 日志打印都有打印.
相同的源码, 相同的java和maven版本, 在windows和ubuntu版本打包, 初始化玩都会在zookeeper中看到节点注册进来, 但是在centos打包的就无法注册zookeeper中, 日志也答应到已经调用初始化方法的了, 将jar包拿回本地运行也无法注册. 因为程序整体执行没问题, 就是elastic-job无法正确注册而已, 请问有什么特殊机制麽?

I suggest you find reason by yourself. no any additional special process here.
Please try to find the reason for different environments.

authors: roaminlove terrymanu

repo: elasticjob/elastic-job-lite


text:
Update dependencies

Hi!
This updates to embedded-hal 1.0.0. A few changes are needed and a good chunk of the tests are still passing, however a few are failing. Maybe someone else can have a look, only the ds3234 tests are failing, seems to be related to SPI.

Thank you for this, I will look into it in the next few days.

authors: DBLouis eldruin


text:
Need to know available SRAM and SDRAM at runtime

[EDITED]
In testing exports to Daisy I discovered 2 issues that call for a runtime method of knowing available space in SRAM and SDRAM.
1.) Using internal SRAM memory is significantly faster than using SDRAM. For example, a reverb patch requiring 240kb of delay memory runs at 42% of available cpu time when using SDRAM memory, but 12% of available cpu time when using SRAM.
2.) If memory allocations using malloc() etc. (which go to SRAM currently) run out of memory, the Daisy appears to simply freeze.
For this reason it would be preferable to have a memory allocator that can use SRAM as much as possible (for speed), and defer to a block of SDRAM when this is exhausted (so that user code does not mysteriously fail).
I was able to hack something along these lines using a pre-allocated block of SDRAM  (thanks to Stephen Hensley for the DSY_SDRAM_BSS macro tip), diverting memory allocations to pull from this block if more than a certain amount has already been allocated from SRAM.  But, this hack is based on guesswork about how much SRAM is available, and isn't really scalable.   It happens to work for gen~ export code and the examples I've tested so far, but it probably won't work for all examples and all platforms and will fail mysteriously.
Knowing available SRAM may not be feasible at compile time, since there may be runtime calls to malloc()/new in Daisy code or binding code, thus pre-assigning RAM sources may not be feasible for user-level code.
Stephen suggested a memory allocator used across libdaisy and projects covering
dsy_malloc(), dsy_free(), dsy_realloc(), dsy_malloc_size()

And the use of these methods throughout codebases that use Daisy. These routines would allocate memory from internal SRAM while such memory is available, since it is significantly faster, then defer to slower SDRAM thereafter; and return freed memory accordingly. Obviously some kind of optimized allocator scheme would be preferred (there are a few open source real-time oriented allocators out there to consider).

And the use of these methods throughout codebases that use Daisy.
No, please not.

This sort of "pseudo-intelligent" memory allocation is not a good idea. Users need to understand which data must go into which RAM and why. With such an automatic malloc you quickly get into situations where the order of operations (= order of memory allocations) has an impact on the performance. This is only asking for trouble.
In most cases, if you use dynamic memory allocation on a bare metal embedded device, you've picked the wrong design pattern. The use cases where dynamic memory allocation is actually needed are very rare.

Not talking about continuous dynamic allocation here (at least in my case for gen~ export there is no memory allocation after the audio starts).
Just talking about the fact that calling malloc() will always go to SRAM, which when full, will simply fail; instead of failing, I think it would be better to start using SDRAM instead in that case. But from private discussions with Stephen the only solution to know when this happens is to count memory allocations, which means applying this rule globally AFAWCT.

there is no memory allocation after the audio starts

As I said, then you probably don't actually need dynamic allocation in the first place.
I quickly looked at some gen~generated C++ and it seems you can allocate it statically, if you're willing to change a few lines of the generated code.

The question is knowing how much SRAM is available, so that allocations can go there as much as possible, since it is 3-4x faster than SDRAM. This value is not known at compile time, since there are also allocations happening within libdaisy etc.

I understand what you are trying to achieve (at least I think). But I don't think it will work the way you expect it to. If you have a malloc that can decide if it allocates from internal or external RAM - that's great. But it won't help you with the gen~ code.
I'm not very familiar with gen~ but from what I've seen, it collects all its state data in a large struct which it creates dynamically with operator new(). Assuming you have your "intelligent" malloc, chances are that your gen~ code is too large to fit into SRAM. As SRAM and SDRAM don't sit in adjacent memory regions, your malloc will have to allocate the whole block for your gen~ code in SDRAM.
Now how does dynamic allocation help you here? All it does for you is:

You don't know at compile time if your large gen~ patch will fit into the RAM at all
You don't know at compile time in which RAM the patch will be allocated (== you can't make any guesses about performance)

If, instead, you modified your gen~ create() function to use placement new() and provided a std::aligned_storage<> as a function argument, then you can place this std::aligned_storage<> in whatever RAM seems appropriate and see right at compile time it it will fit.

Hi thanks for the detailed attention!
gen~ data isn't all stored in a large struct. The majority of parameter variables, filter state variables, latched variables, etc. are local to the struct, as they are likely to be referenced continously and are thus prioritized in memory location, but any large items (which means only delay lines and sample buffers) are allocated separately as they are more likely to be accessed randomly.  There's almost no chance that the main struct will be too big for SRAM (I seem to be able to get away with about 240Kb, which I'm pretty confident is more than enough for even an extremely complex patch). The buffers and delays get allocated separately to this struct, and at a later stage of initialization.
It's just a question of where these buffers and delays get allocated. Ideally, as many that can fit into SRAM as possible, for better performance, hence the question.

In this case, your best bet is to write your own allocator that "allocates" memory from a fixed size block that you place in whatever RAM you want. At least you can check at runtime that your block is large enough and - if not - blink a user led and abort.

That's exactly what I have working already for SDRAM (which has to be pre-allocated anyway). It's just that I'd rather use as much SRAM as I can before doing so, since it is 3-4x faster. Hence needing a way of knowing how much SRAM remains available.
(For reference, a Dattoro style reverb can fit entirely in 230Kb of SRAM, but the Gigaverb is much larger, needing 1632Kb -- or 240Kb of SRAM and 1402Kb of SDRAM, which also works, and saves a significant % of cpu in audio callback performance compared to allocating all delays in SDRAM. But honestly picking 230Kb as my threshold is just guesswork.)

gen~ data isn't all stored in a large struct. The majority of parameter variables, filter state variables, latched variables, etc. are local to the struct, as they are likely to be referenced continously and are thus prioritized in memory location, but any large items (which means only delay lines and sample buffers) are allocated separately as they are more likely to be accessed randomly.

Thanks for the clarification. In this case, you'll have to live with the dynamic allocation. Too bad.

It's just that I'd rather use as much SRAM as I can before doing so

No one stops you from having two preallocated blocks - one in SRAM, one in SDRAM - and using these in your custom allocator. You only need a single head index that contains the index of the first unused byte in your two blocks. For each allocation, you check if head + numBytes <= sram_size and if so, "allocate" from the SRAM block, otherwise from the SDRAM block. Remember the alignment requirements.
At this point, we're talking about how to supply memory to gen~. My original point is still valid: You don't actually need true dynamic allocation. Your custom allocator can be used for gen~ only, and it can enter a fault state (= blink user LED, etc.) if it overflows. You should keep the rest of the system as static as possible to avoid all the trouble that comes with dynamic allocation.
Maybe I should rephrase my main concerns: If we add an intelligent malloc() to libDaisy, we encourage avoidable dynamic allocation and cause more problems than we solve. Keeping malloc() intentionally dumb forces users to at least think about how the memory model should work. We can still add custom allocators if needed, but they should come with big warning signs.

Interesting issue that we will have to think about also with Faust.
In Faust DSP are basically a class where everything (controllers, delays line...) is currently allocated at the same level. We may want to refine this a bit and use a 2 level memory layout in SRAM and SDRAM as explained by @grrrwaaa with gen~.
Since several projects have the same needs, what could be interesting to have libdaisy provide some generic memory primitives to allocate separately in SRAM and  SDRAM.

Thanks for the clarification. In this case, you'll have to live with the dynamic allocation. Too bad.

I don't see it as bad :-) ... it's more flexible (and was designed to be, since gen~ targets desktop as well) -- we get to insert whatever allocation scheme we need by overriding the genlib_malloc calls that all the exported code goes through -- and more importantly, we can separate allocation of larger random-access chunks like buffers from tighter-loop persistent data, which I think will work very well for the Daisy platform.
If nothing else, perhaps this is what libDaisy might also do -- have a dsy_malloc which is initially #defined to malloc, but can be replaced. I notice that some of the libraries embedded in libdaisy have this equivalent (ff_malloc, usbd_malloc, etc.). That would allow injection of allocators by platforms as needed but will perhaps allay concerns about encouraging dynamic allocation.
Returning to my specific problem: In my Daisy work I do already pre-allocate memory stores in SRAM and SDRAM and have a custom allocator draw from those -- it does work (and gets me significantly better performance than naively allocating all buffers from SDRAM), but my problem is that I don't know how much SRAM I can safely reserve for the user-level gen~ code, which makes me worried that this custom allocator may be liable to break unrepdictably. I tried a quick test by attempting to malloc (and then free) blocks of 1Kb until malloc returns null -- at the start of main() it reports 511Kb available by this method. Unfortunately this test isn't viable in practice as it also seems to crash the Daisy thereafter.
As @sletz notes, this need to know available memory and to allocate in SRAM when possible and defer to SDRAM when needed probably isn't specific to gen~ and will likely be something that other projects will also want, thus might warrant being written once within libdaisy where it can benefit from community input, rather than individually for each project.

have a dsy_malloc which is initially #defined to malloc, but can be replaced.

I would prefer weak linkage, but yes, that could be a solution.

I have today tried to get C++ code generated from the SOUL language to run on the Daisy - and compare it to handwritten C++. Fun fact: I can't reproduce your results regarding the "3-4x" speed difference. I see more like 8% increase in CPU render time. At the same time, the code seems way slower than I would assume - the simple reverb I used is barely realtime capable. It all seems fishy. If you want to take a look, see this (includes the code that I used): https://github.com/soul-lang/SOUL/issues/33#issuecomment-670977536
Weird thing is: I've heard similar speed difference numbers stated by other people so I'm starting to think my Daisy is not running at the full speed for some reason. Maybe my libDaisy is tweaked to death? The link above contains my code, if you could run it on your Daisy and let us know the speed differences, that would be very interesting to see.

SDRAM performance is extremely dependent on access patterns. If you access data more or less linearly and keep within the constraints of DCACHE it can be pretty fast. Any sort of random access pattern can be more than 10x slower than DTCM.  Most reverb implementations are generally very bad for cache coherency as they access a large number of samples from a wide range of addresses to compute one output sample. Its possible but pretty tricky to re-order reverb delay accesses to be more linear.
We use an allocator that allows us to choose which of the 9 types of H7 ram an allocation will occur from and there is a "pick the fastest ram the allocation will fit in" option as well (fastest with respect to cpu access). We do not support freeing of ram (or use of any general purpose heap allocation). We have templatized versions that support typical C++ type initialization (constructors) as well. The trickiest part is the interface to the linker as you need to know that you are not allocating from an address that the linker has put something else in already.
auto* mem = mem_h7::get();
verb	  = mem->alloc<verb_t>(H7_DTCM_D0);
delays[0]  = mem->alloc_fast<float, delay_count>();
delays[1]  = mem->alloc_fast<float, delay_count>();
...

 Knowing available SRAM and SDRAM at runtime

authors: Stephenmccaul TheSlowGrowth andrewikenberry grrrwaaa sletz

repo: electro-smith/libDaisy


text:
Question : is the native remote sync limitation still there ?

I'm not sure but I guess https://github.com/electron/electron/pull/2391 has made the remote async using Promise, hasn't it ?

Same question for @paulcbetts and @CharlieHess, if still active on this project...

authors: cweekly mmouterde

repo: electron-userland/electron-remote


text:
Could not establish secure connection

Hello everyone. When I'm trying to notarize my app in teamcity, I caught a error. But When I'm trying to do this locally, everything is good.
I think the error is due to the certificate. How can I substitute a certificate?
Domain is api.apple-cloudkit.com
Certificate trust evaluation did not return expected result. (5)  [leaf AnchorApple ChainLength IntermediateMarkerOid LeafMarkersProdAndQA]
Certificate trust evaluation for api.apple-cloudkit.com did not return expected result. Missing required extension.
Certificate trust evaluation did not return expected result. (5)  [leaf AnchorApple ChainLength IntermediateMarkerOid LeafMarkersProdAndQA]
Certificate trust evaluation for api.apple-cloudkit.com did not return expected result. Missing required extension.
Could not establish secure connection to api.apple-cloudkit.com

The settings

I have the same error,  do you solve it ?

Maybe you should close your NET Proxy

authors: denkotheboy sunnysxr

repo: electron/notarize


text:
I have a problem

May I ask the author, where is the matrix used to calculate the terminal error, stage cost, and input cost? May I ask
why  mterm = self.model.aux['cost']  # Terminal cost and lterm = self.model.aux['cost']  # Stage cost  use "cost".

May I ask the selection basis of # State cost matrix and # Controls cost matrix and why there is no terminal error matrix?

The objective function is quadratic so the stage cost is defined by the self.Q and self.R matrices.
Using the same weight matrix Q for both the stage and terminal costs is a design choice. If you wanted to use a different terminal cost, you could for example set mterm=transpose(X)@Q_final@X.
In applications where the system does not reach the final goal within the prediction horizon, it does not make much sense to penalize arbitrary intermediate states more heavily since they are not part of the goal set. That is why I chose to use the same weight matrix.

May I ask whether the author set the error weighting sum of the terminal error and the input weighting sum in the following code, and set the matrix of the error weighting sum (state variable weight) and the matrix of the terminal error (state variable terminal weight matrix) to be the same?
mterm = self.model.aux['cost']  # Terminal cost
lterm = self.model.aux['cost']  # Stage cost
mpc.set_objective(mterm=mterm, lterm=lterm)
mpc.set_rterm(u=self.R)  # Input penalty (R diagonal matrix in objective fun)

authors: elena-ecn guest-oo

repo: elena-ecn/mpc-cbf


text:
🛑 UNER - Guaraní is down

In 87a6f7e, UNER - Guaraní (https://g3autogestion.uner.edu.ar/g3w3/) was down:

HTTP code: 0
Response time: 0 ms

Resolved: UNER - Guaraní is back up in dd4819c.

repo: elfoche/monitoreo


text:
🛑 MCU - Owncloud is down

In ff95cd5, MCU - Owncloud (https://cloud.cdeluruguay.gob.ar/login?redirect_url=/apps/dashboard/) was down:

HTTP code: 0
Response time: 0 ms

Resolved: MCU - Owncloud is back up in de51d81.


text:
Elementum 0.0.46 buffers extremely slow, if it does so at all, and never actually plays a torrent.

Hello,
I have been trying to get Elementum 0.0.46 to work for quite a while on my NVIDIA Shield TV; however, I have never any luck in doing so. I initially tried Quasar and it seemed to download a torrent extremely quick, but the torrent itself never played either. I have enabled remote control in Kodi, tried all different storage types in settings, uninstalling/reinstalling, and searched the web tirelessly for a solution to no avail.
What tends to happen with Elementum is that it will begin buffering (stay on 0% for about 5 minutes) then state "Playback failed" then begin buffering (at a painfully slow speed <200kbps) then state "Playback failed" again. Bear in mind, my internet speed over my VPN is roughly 80mbps/20mbps. Although Quasar never successfully played a torrent either and presented me with the same "Playback Failed" message, the buffering is consistently >1500kbps, and never takes more than five seconds to begin buffering. I tried this test on identical torrents within five minutes of each other (Elementum and Quasar weren't simultaneously installed) on different occasions.
I have pasted my Kodi log below for Elementum, and for Quasar if that would help fix the speed issue in Elementum. Any help would be much appreciated!
Your Environment
Elementum 0.0.46
Kodi 17.6
NVIDIA Shield TV (Android 7/SHIELD Android TV SW Version 6.3)
Elementum Log: https://paste.ubuntu.com/p/rvnjXY6F4m/
Quasar Log: https://paste.ubuntu.com/p/snNXKBJFGW/

I am seeing a similar issue on my Vero 4k. Torrents all stay around 0kb/s. Used to work fine and I'm certain that the connection is solid.
Elementum 0.0.46
Kodi Debug Logs: https://paste.ubuntu.com/p/3XPsckvQPj/

Hi Strummer, that is interesting you are facing a similar issue. Unfortunately, I have never gotten it to work successfully even in version 0.0.45.

@nvpl  can you try memory storage, not using files?
@strummerj I need full log, not a small cut.

Thank you @elgatito  for your response. I tried all four file storage types in the past with no success. Nonetheless, I went ahead and tried again with "memory storage" with no luck.
Pasted below is my log when using "memory storage."
https://paste.ubuntu.com/p/WsZBrYj6BM/

@nvpl very strange log. Can you open http://KODI_IP:65520/info url and save the page? It will show torrent library information.

@elgatito I believe you meant to say: http://KODI_IP:65220/info instead of http://KODI_IP:65520/info
Here is the log/torrent library information about 30 seconds after I clicked on the torrent: https://paste.ubuntu.com/p/ynw7B54d6H/
Here is the log/torrent library information after the "Playback failed" message displayed: https://paste.ubuntu.com/p/GW3BKR3QXs/

@nvpl very strange log. Torrent information looks good to me and there are active peers, but download is so slow.
casting @anacrolix help :)

It would appear the network is going very slowly. The playback failed is very likely a timeout occurring due to a timeout from Kodi. At the time that the error occurs, there's only a single peer uploading to you with only 75KiB/s. There appears to be plenty of peers, it's strange there's only 10 connected, and they're all TCP. They're also all outbound, meaning connections likely aren't being received (due to firewall or NAT).
I'd go through the following checklist:

Ensure global connections/connections per torrent is at least 30
Do a network performance test (speedtest.net or something similar)
Tell us your ISP, and cable/ADSL whatever, it may be your traffic is being shaped or tampered with

The fact that other plugins have the same issue suggests that either your network, or something else with your setup is bad.

@anacrolix

Upon checking my settings under Elementum > Addon Settings > BitTorrent > Connections limit, there is no limit specified for "Connections limit." Am I referencing the correct option?
Speedtest over VPN from NVIDIA Shield TV pasted below.
I am using cable internet. My NVDIA Shield is hard wired to the router with OpenVPN running in the background. Using the same VPN and network, I am able to download torrents on my laptop over wifi at ~ 7mbps. Which causes me to doubt that my ISP tampering with my network traffic.
I don't know if the issue could be network related as I have no issue torrenting on other devices on the same network; however, I am no expert. Other plugins such as Quasar do not experience the same issue entirely because that plugin reaches download speeds similar to what I am able to achieve on my laptop while torrenting. However, I still receive the playback failed message on Quasar.

@elgatito will know the connection limit set. I don't think that's it if you can torrent otherwise as you say. Speeds look fine. The main reason speeds might appear lower with elementum is the torrent package is a lot more selective and conservative about what it downloads than the implementations other clients use, so I think that's to be expected unless you run them neck and neck. I think the issue lies with Quasar on your system which isn't an area I'm familiar with.

@anacrolix
EstablishedConnsPerTorrent:0, HalfOpenConnsPerTorrent:0, TorrentPeersHighWater:0, TorrentPeersLowWater:0, HandshakesTimeout:0

We don't set any max/min options by default. This one also has 0 settings.

@elgatito Apologies for all the different logs ahead of time. I figured I should be provide as much info as possible with the new settings applied/new version of Elementum being released.
Logs—version 0.0.46 after disabling "Auto-assign network IP" and "Listen interfaces" set to "tun1":

Info page ~ 1 minute after torrent clicked: https://paste.ubuntu.com/p/9m5VYCt4Tz/

Info page immediately after "Playback failed"/"Elementum timed out" message: https://paste.ubuntu.com/p/4B45grWXKd/

Debug Page ~ 1 minute after: https://paste.ubuntu.com/p/Hr2YR5J88Y/

Debug page immediately after "Playback failed"/"Elementum timed out" message: https://paste.ubuntu.com/p/sHVQWtw5mv/

Logs—version 0.0.46 after disabling "Auto-assign network IP" and "Listen interfaces" set to "tun0":

Info page ~ 1 minute after torrent clicked:  https://paste.ubuntu.com/p/csPbYdpKRS/

Info page immediately after "Playback failed"/"Elementum timed out" message: https://paste.ubuntu.com/p/nYRCW3nSGs/

Debug Page ~ 1 minute after: https://paste.ubuntu.com/p/YfW4JccDyr/

Debug page immediately after "Playback failed"/"Elementum timed out" message: https://paste.ubuntu.com/p/T6rXRF35dj/

Logs—version 0.0.47 after disabling "Auto-assign network IP" and "Listen interfaces" set to "tun0":

Info Page ~ 1 minute after torrent clicked:  https://paste.ubuntu.com/p/7zRCY7yp95/

Info Page immediately after "Playback failed"/"Elementum timed out" message: https://paste.ubuntu.com/p/TYDsbBDKDY/

Debug Page ~ 1 minute after torrent clicked: https://paste.ubuntu.com/p/PjqmSJZKnk/

Debug Page immediately after "Playback failed"/"Elementum timed out" message: https://paste.ubuntu.com/p/kmPHx9yq4W/

LockTimes Page ~ 1 minute after torrent clicked: https://paste.ubuntu.com/p/XpffNnWcjH/

LockTimes Page immediately after "Playback failed"/"Elementum timed out" message: https://paste.ubuntu.com/p/s2Vw6zw7PQ/

Moreover, this is not an issue for me at all but I thought I would mention this since it may be related. Upon first installing the last three versions of Elementum (0.0.45/46/47), it crashed about five times before the plugin decided to just not start at all. Once I exit/reenter Kodi though, the plugin acts perfectly normal (other than the preexisting issues being addressed here). In other words, the crashing only happens upon initial installation and never again after the fact.

Elementum 0.0.47 installation crash log—(debug page loaded while Elementum was in the middle of crashing/restarting): https://paste.ubuntu.com/p/ck6ChsRjF4/

@nvpl , great, you've posted all these logs, but kodi.log is very needed to see what happens in that moment, when you open /debug/ links. Because I don't even see how speed changes with different test environments.

@elgatito I completely forgot about those logs. I am not too familiar with how the logs work; however, to my understanding Kodi creates a new log every time I exit the application. So  I will attach the last two logs I have from Kodi which I assume correspond to the initial installation/crashing of the new 0.0.47 version (Kodi old log) and test performed in the previous message 0.0.47 with "Listen interfaces" set to "tun0" (Kodi new log) respectively.
Kodi old log: https://paste.ubuntu.com/p/jShRdXB5qR/
Kodi new log: https://paste.ubuntu.com/p/7htd6C5XqQ/
Thanks again.

I don't see anything unexpected in the pages or logs from the torrent perspective.

@nvpl can you try with the latest version, setting listen interface to "tun0" (if it's tun0), run download for about a minute. Then, set connections limit (bittorrent settings tab) to about 30, run another download. And then paste full kodi.log (it can be one for all tests).
Also can you watch CPU usage while download is in progress?

@elgatito

How can I tell which listen interface is the correct one to use?

Kodi Log with Listen interface set to "tun0" on latest version (0.0.47 both tests combined in one log): https://paste.ubuntu.com/p/cnKcNP49v8/
Elementum Info page with connections limit set to 30 on "tun0" listen interface: https://paste.ubuntu.com/p/txrP88RvtJ/
Elementum Debug page with connections limit set to 30 on "tun0" listen interface: https://paste.ubuntu.com/p/dYGp56Xn5t/
Link to video of CPU usage: https://streamable.com/ixut1

@elgatito Okay, I have removed connections limit and cleared the "listen interface" but left "auto-assign network IP" off. Feel free to review the video below to make sure my BitTorrent settings tab looks correct. If it makes any difference, while I was installing version 0.0.48 and the initial crashing upon installation occurred again which should be included in the bundle log.
Debug Bundle Log During Download ~ 1 minute: https://paste.ubuntu.com/p/HGp4DmXZNy/
Debug Bundle Log After Clicking Cancel and ~ 1 minute of downloading: https://paste.ubuntu.com/p/qRxpkmJfjK/
Video (CPU Usage/Settings): https://streamable.com/g3q1p

Thanks for providing the logs.
Have you tried non rarbg torrents? Another tracker/webite?

@elgatito I don't believe I've been selecting ragbg's torrents. I've been using YTS in my video tests. However, I have tried other uploaders as well. If you'd like to provide me with a sample torrent to download I can try that as well.
I just performed another test from a different uploader (MagnetDL) yielding similar results to the rest. Here is the log: https://paste.ubuntu.com/p/hkFYKDQfMc/

@elgatito I wanted to check in and see if I provided the correct logs/see if there is any update regarding this issue.
Thanks again.

22:49:41.738 Buffer loop
22:49:42.675 3f2a44: 36 kB/0 B | Buffering (0.01%)
22:49:43.676 3f2a44: 246 kB/0 B | Buffering (0.08%)
22:49:44.677 3f2a44: 891 kB/0 B | Buffering (0.30%)
22:49:45.676 3f2a44: 1.8 MB/0 B | Buffering (0.61%)
22:49:46.678 3f2a44: 2.0 MB/0 B | Buffering (0.71%)
22:49:47.675 3f2a44: 2.3 MB/0 B | Buffering (0.86%)
22:49:48.676 3f2a44: 1.8 MB/0 B | Buffering (0.91%)
22:49:49.676 3f2a44: 904 kB/0 B | Buffering (0.91%)
22:49:50.679 3f2a44: 626 kB/0 B | Buffering (0.91%)
22:49:51.679 3f2a44: 164 kB/0 B | Buffering (0.91%)
22:49:52.676 3f2a44: 0 B/0 B | Buffering (0.91%)
22:49:53.676 3f2a44: 0 B/0 B | Buffering (0.91%)

So the speed first was good, then dropped to 0, then upped to 200kb/s after about 30 seconds.
For me it still looks like a network or firewalling issue...
Can you try to set connections limit to 20, start a download, open http://127.0.0.1:65220/info after few seconds, then after another minute.

Not important since the issue doesn't persist after initial installation but, I am pasting this debug bundle installation log which contains the initial crashes here again since you released a new version (0.0.49): https://paste.ubuntu.com/p/Sf4mRRMYft/
Anyways, here is what you requested:
Connections Limit Set to none

Elementum Info Page after ~ 20 Seconds: https://paste.ubuntu.com/p/8QRDmjp8q9/

Debug Bundle after ~ 20 Seconds: https://paste.ubuntu.com/p/fFfrWsBy5J/

Elementum Info Page after ~ 1 minute: https://paste.ubuntu.com/p/Zj7b3MknFY/

Debug Bundle after ~ 1 minute: https://paste.ubuntu.com/p/VGsSzFqgY2/

Connections Limit Set to 20

Elementum Info Page after ~ 20 Seconds: https://paste.ubuntu.com/p/KgvYBHVrYk/

Debug Bundle after ~ 20 Seconds: https://paste.ubuntu.com/p/ZQkDQchg2W/

Elementum Info Page after ~ 1 minute: https://paste.ubuntu.com/p/3NrdW6Ncs9/

Debug Bundle after ~ 1 minute: https://paste.ubuntu.com/p/htzpGH8BQy/

It seems that the buffer did not go above 0 kbps when the connections limit was set to 20. I have pasted logs for both situations using identical torrents.

@anacrolix what do you think?
From the last log (so it's about a minute after start):
 TotalPeers: (int) 502,
 PendingPeers: (int) 497,
 ActivePeers: (int) 0,
 ConnectedSeeders: (int) 0,
 HalfOpenPeers: (int) 10

Hey @anacrolix, any new insight in regards to @elgatito's message?
Some more info: I tried switching off my VPN to see if that would fix the issue but I received the exact same results. I also tried using my mobile hotspot rather than my home internet without a VPN which also yielded the same results. Hence, I highly doubt it is a firewall issue.

I'm not sure. I'm suspecting maybe dial timeouts are too high and there are too many bad peers to work through. I'll await @elgatito 's next update, where a lot more information is exposed for debugging.

@nvpl do you still have the issue? Can you test the latest version?

Hi @elgatito,
I apologize for the late response.
I have updated to version 0.0.53 and the speeds seem to have become somewhat functional. Yet, it still pales in comparison to Quasar. Additionally, I think I've finally found the reason as to why the torrent itself never plays (as shown in video) even after buffering. This seems to be a compatibility issue that both Elementum and Quasar have with NVIDIA Shield TVs and official Kodi. The reason I have come to this conclusion is because I installed SPMC (a modified Kodi for Android based systems) and tried Elementum and Quasar on there and both plugins play torrents successfully. However, Quasar buffers at nearly 5x the speed that Elementum does. I would much prefer to use Elementum though because it is being regularly updated. Quasar also has an issue where the video frame stutters every ~3-5 seconds rendering it unusable to watch movies on.
In other words as of version 0.0.53,

Elementum's speed issue seems to have been fixed to the point that it will complete a buffer (though it won't go above ~1500kbps in official Kodi).
If using official Kodi from Google Play Store in conjunction with Elementum, playing torrents still fails after the torrent fully buffers.
If using SPMC 17.6a2 on NVIDIA Shield TV in conjunction with Elementum (with identical settings), buffering speeds are slightly improved (~2000kbps) in comparison to official Kodi; however, still slower than Quasar. Most importantly, the torrent will complete the buffer and play (shown in video below).
When looking at speeds of Quasar running for a minute on the webUI, the speeds stay pretty stable at around ~7000kbps
When looking at speeds of Elementum running for a minute on the webUI, the speeds heavily fluctuate after reaching a peak of 2000kbps during buffering then dropping down to 400kbps to 1kbps and kind of float between 1-400kbps for the rest of the time during the video playing.

Screenshot of Quasar speeds after 1 minute (webUI):

Video displaying my current settings in Quasar (SPMC) along with the buffer and successful play (take note of the significantly higher speeds on the same torrent): https://streamable.com/3qpe6

Screenshot of Elementum speeds after 1 minute (webUI):

Video displaying my current settings in Elementum (SPMC) along with the buffer and successful play: https://streamable.com/4r6l9

Elementum log from /debug/all after one minute in official Kodi (video fails to play): https://paste.ubuntu.com/p/3kZvdrBVdF/
Elementum log from /debug/all after one minute in SPMC (video successfully plays): https://paste.ubuntu.com/p/2Ws7Qw7FDD/
SPMC debug log Quasar (I used SPMC debug because I don't know where to find debug logs in Quasar): https://paste.ubuntu.com/p/VKXS4GphMK/
Video displaying my current settings in Elementum (Kodi official) along with the buffer and failure to play: https://streamable.com/d3y96

@nvpl thanks for such detailed information.
When you set memory storage - it downloads only few pieces of file, then download more when needed from Kodi, so you won't see the best speeds and it drops before it gets to the maximum. You can try to set about 300mb per torrent, to see bigger speeds. While you get all you need - speed will be about 0 kb/s, then again rising and falling and so on.
@anacrolix do you anything useful in /debug/all dumps? writeChunk speed is good, maxxed at 5ms, but this video it strange - https://streamable.com/4r6l9 , there are so many peers and enough connected, and no write issues, but so small speed and so slowly raising.

@nvpl can you send the kodi.log (of one from /debug/bundle) where playback can't be started with elementum?

I notice no incoming connections complete handshaking, and there's no incoming TCP connections at all. Everything else looks good.

There's something really going on with number of connections. I've got it set to 15, but sometimes elementum bursts out with connections:
conntrack -L | wc -l conntrack v1.4.4 (conntrack-tools): 58687 flow entries have been shown. 58687
to the point it can no longer make a connection
Aug 31 15:59:43 host kernel: [1436784.235951] nf_conntrack: nf_conntrack: table full, dropping packet
Hopefully it is related to https://github.com/anacrolix/torrent/issues/234 and will probably be resolved some day.
My ISP goes crazy with a number of connections like this.

@anacrolix Thank you. But it looks like other torrent clients don't do this at the rate as anacrolix torrent does. This is really problem for, say, routers with NAT or mobile network providers with grey addresses and NAT. Routers surely have linux with conntrack there. It would be nice to have an ability to throttle down these connections.  Because currently it renders unusable anacrolix client for any user behind NAT.
I would love global connections limit to be applied to DHT connections also. Default nf_conntrack_udp_timeout is set to 30. It would be nice to implement a way to count active and inactive DHT connections and keep them under connection limit.

Please, try with 0.0.63.

@nvpl @strummerj do you still have this issue with the latest Elementum?

authors: anacrolix dadittoz elgatito nvpl strummerj


text:
MSID: 91783 Version: 1 DOI: 2023.09.14.557527

"msas": "Cell Biology", "Computational and Systems Biology"
"msid": "91783"
"preprintDoi": "10.1101/2023.09.14.557527"
"Reviewed Preprint posted": "2023-11-13"
"Sent for peer review": "2023-08-25"
"Posted to bioRxiv": "2023-09-14" (link: "Go to bioRxiv": "https://www.biorxiv.org/content/10.1101/2023.09.14.557527v1/")
[PLACE PDF URL HERE WHEN AVAILABLE] See step 7
Step 1. Inform bioRxiv
Who can help: @QueenKraken, @nlisgo, @scottaubrey

[x] Manuscript is in data hub index (https://data-hub-api.elifesciences.org/enhanced-preprints/docmaps/v2/by-publisher/elife/get-by-manuscript-id?manuscript_id=91783)

Step 2. Create preview of manuscript
Who can help: @fred-atherden, @nlisgo, @scottaubrey

[ ] Manuscript is available for preview (https://prod--epp.elifesciences.org/preview/91783/v1)

[ ] Trigger import for staging: https://prod--epp.elifesciences.org/import
[ ] Sync data folder with s3://prod-elife-epp-data/data
[ ] Trigger import for staging: https://staging--epp.elifesciences.org/import
[ ] Sync data folder with s3://staging-elife-epp-data/data
[ ] Pull request created to enhanced-preprint-data
[ ] Post link to pull request below

Pull request: [PLACE LINK TO PULL REQUEST HERE]
Detailed instructions: https://github.com/elifesciences/enhanced-preprints-data#add-a-manuscript
Step 3: Awaiting public reviews and QC
Who can help: Production team

[ ] QC passed or QC not needed

Who can help: Editorial team

[x] Public reviews available on sciety (https://sciety.org/articles/activity/10.1101/2023.09.14.557527)
[x] Manuscript and editor details at the top of this issue (Supplied by Editorial team)
[x] Reviews returned by EPP - https://prod--epp.elifesciences.org/api/reviewed-preprints/91783/v1/reviews

Example
"msas": "Genetics and Genomics", "Neuroscience"
"msid": "84628"
"version": "1"
"preprintDoi": "10.1101/2022.10.28.514241"
"articleType": "Reviewed Preprint"
"status": "Published from the original preprint after peer review and assessment by eLife."

"Reviewed Preprint posted": "2023-01-02"
"Sent for peer review": "2022-10-28"
"Posted to bioRxiv": "2022-11-21" (link: "Go to bioRxiv": "https://www.biorxiv.org/content/10.1101/2022.10.28.514241v1")

Editors:
Reviewing Editor
Michael B Eisen
University of California, Berkeley, United States
Senior Editor
Michael B Eisen
University of California, Berkeley, United States

Step 4: Modify manuscripts.json (no PDF)

[ ] Reviewed preprint is published to EPP (https://elifesciences.org/reviewed-preprints/91783)

[ ] Pull request created to enhanced-preprint-client
[ ] Pull request switched to "Ready for review"
[ ] Post link to pull request below

[ ] Request that a doi be registered

[ ] Post link to #enhanced-preprint comment below

Pull request: [PLACE LINK TO PULL REQUEST HERE]
#enhanced-preprint comment thread: [PLACE LINK TO COMMENT HERE]

Instructions to modify manuscripts.json

Visit: https://github.com/elifesciences/enhanced-preprints-client/actions/workflows/publish-manuscript.yaml
Click: Run workflow
Complete the form and click "Run workflow"
A successful run should result in a new pull request at https://github.com/elifesciences/enhanced-preprints-client/pulls
Open the pull request and click the "Ready for review" button to trigger tests
Once the tests pass and you are happy with the changes the PR can be merged

Example pull request: https://github.com/elifesciences/enhanced-preprints-client/pull/334/files
Once the pull request is merged in it should be available a few minutes later.

Request that a doi
Post the following in #enhanced-preprint:
@Fred can you register a doi for https://elifesciences.org/reviewed-preprints/91783

Step 5: Awaiting search reindex

[ ] Reviewed preprint is avaliable on journal homepage (https://elifesciences.org)

The search reindex is triggered once an hour. We need the reviewed preprint to be indexed as the search application serves the journal homepage.

Additional info
If needed, the jenkins pipeline to reindex search can be triggered sooner.
https://alfred.elifesciences.org/job/process/job/process-reindex-reviewed-preprints/

Step 6: Published! Request PDF generation

[ ] Post the link to the PDF url at the top of the issue

[ ] Post link to #enhanced-preprint comment below

#enhanced-preprint comment thread: [PLACE LINK TO COMMENT HERE]
Post the following to the #enhanced-preprint on slack:
@Ryan Dix-Peek please can you generate a PDF for https://elifesciences.org/reviewed-preprints/91783

Step 7: Introduce PDF to data folder and git repo

[ ] PDF is avaliable at https://github.com/elifesciences/enhanced-preprints-data/raw/master/data/91783/v1/91783-v1.pdf

Detailed instructions: https://github.com/elifesciences/enhanced-preprints-data#add-a-pdf
Step 8: Add PDF url to manuscripts.json

[ ] Reviewed preprint PDF is available for download (https://prod--epp.elifesciences.org/reviewed-preprints/91783)

[ ] Pull request created to enhanced-preprint-client
[ ] Pull request switched to "Ready for review"
[ ] Post link to pull request below

[PLACE LINK TO PULL REQUEST HERE]

Instructions to add PDF url to manuscripts.json

Visit: https://github.com/elifesciences/enhanced-preprints-client/actions/workflows/add-pdf-url-to-manuscript.yaml
Click: Run workflow
Complete the form and click "Run workflow"
A successful run should result in a new pull request at https://github.com/elifesciences/enhanced-preprints-client/pulls
Open the pull request and click the "Ready for review" button to trigger tests
Once the tests pass and you are happy with the changes the PR can be merged

Example pull request: https://github.com/elifesciences/enhanced-preprints-client/pull/397/files
Once the pull request is merged in it should be available a few minutes later.

Step 9: Done!

[ ] Kettle is on!

Emails need fixing, otherwise QC OK

Package uploaded to s3://prod-elife-epp-meca/91783-v1-meca.zip

authors: fred-atherden nlisgo

repo: elifesciences/enhanced-preprints-import