2020-05-14 18:34:18 +10:00
|
|
|
/*
|
2021-04-28 22:46:44 +02:00
|
|
|
* Copyright (c) 2020, the SerenityOS developers.
|
2020-05-14 18:34:18 +10:00
|
|
|
*
|
2021-04-22 01:24:48 -07:00
|
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
2020-05-14 18:34:18 +10:00
|
|
|
*/
|
|
|
|
|
|
2021-01-15 21:46:23 +01:00
|
|
|
#include <AK/Debug.h>
|
2020-05-14 18:34:18 +10:00
|
|
|
#include <LibGemini/GeminiResponse.h>
|
|
|
|
|
#include <LibGemini/Job.h>
|
|
|
|
|
#include <stdio.h>
|
|
|
|
|
#include <unistd.h>
|
|
|
|
|
|
|
|
|
|
namespace Gemini {
|
|
|
|
|
|
ProtocolServer: Stream the downloaded data if possible
This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```
Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.
2020-12-26 17:14:12 +03:30
|
|
|
Job::Job(const GeminiRequest& request, OutputStream& output_stream)
|
|
|
|
|
: Core::NetworkJob(output_stream)
|
|
|
|
|
, m_request(request)
|
2020-05-14 18:34:18 +10:00
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Job::~Job()
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
ProtocolServer: Stream the downloaded data if possible
This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```
Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.
2020-12-26 17:14:12 +03:30
|
|
|
void Job::flush_received_buffers()
|
|
|
|
|
{
|
|
|
|
|
for (size_t i = 0; i < m_received_buffers.size(); ++i) {
|
|
|
|
|
auto& payload = m_received_buffers[i];
|
|
|
|
|
auto written = do_write(payload);
|
|
|
|
|
m_received_size -= written;
|
|
|
|
|
if (written == payload.size()) {
|
|
|
|
|
// FIXME: Make this a take-first-friendly object?
|
|
|
|
|
m_received_buffers.take_first();
|
|
|
|
|
continue;
|
|
|
|
|
}
|
2021-02-23 20:42:32 +01:00
|
|
|
VERIFY(written < payload.size());
|
ProtocolServer: Stream the downloaded data if possible
This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```
Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.
2020-12-26 17:14:12 +03:30
|
|
|
payload = payload.slice(written, payload.size() - written);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-14 18:34:18 +10:00
|
|
|
void Job::on_socket_connected()
|
|
|
|
|
{
|
|
|
|
|
register_on_ready_to_write([this] {
|
|
|
|
|
if (m_sent_data)
|
|
|
|
|
return;
|
|
|
|
|
m_sent_data = true;
|
|
|
|
|
auto raw_request = m_request.to_raw_request();
|
2021-01-15 21:46:23 +01:00
|
|
|
|
2021-01-23 23:59:27 +01:00
|
|
|
if constexpr (JOB_DEBUG) {
|
2021-01-15 21:46:23 +01:00
|
|
|
dbgln("Job: raw_request:");
|
|
|
|
|
dbgln("{}", String::copy(raw_request));
|
|
|
|
|
}
|
2020-05-14 18:34:18 +10:00
|
|
|
bool success = write(raw_request);
|
|
|
|
|
if (!success)
|
2021-08-30 18:12:48 +00:00
|
|
|
deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
|
2020-05-14 18:34:18 +10:00
|
|
|
});
|
|
|
|
|
register_on_ready_to_read([this] {
|
|
|
|
|
if (is_cancelled())
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
if (m_state == State::InStatus) {
|
|
|
|
|
if (!can_read_line())
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
auto line = read_line(PAGE_SIZE);
|
|
|
|
|
if (line.is_null()) {
|
2021-05-31 15:02:30 +01:00
|
|
|
warnln("Job: Expected status line");
|
2021-08-30 18:12:48 +00:00
|
|
|
return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
|
2020-05-14 18:34:18 +10:00
|
|
|
}
|
|
|
|
|
|
2020-12-13 11:44:53 +01:00
|
|
|
auto parts = line.split_limit(' ', 2);
|
2020-05-14 18:34:18 +10:00
|
|
|
if (parts.size() != 2) {
|
2020-12-13 11:44:53 +01:00
|
|
|
warnln("Job: Expected 2-part status line, got '{}'", line);
|
2021-08-30 18:12:48 +00:00
|
|
|
return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
|
2020-05-14 18:34:18 +10:00
|
|
|
}
|
|
|
|
|
|
2020-06-12 21:07:52 +02:00
|
|
|
auto status = parts[0].to_uint();
|
|
|
|
|
if (!status.has_value()) {
|
2021-05-31 15:02:30 +01:00
|
|
|
warnln("Job: Expected numeric status code");
|
2021-08-30 18:12:48 +00:00
|
|
|
return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
|
2020-05-14 18:34:18 +10:00
|
|
|
}
|
|
|
|
|
|
2020-06-12 21:07:52 +02:00
|
|
|
m_status = status.value();
|
2020-05-14 18:34:18 +10:00
|
|
|
m_meta = parts[1];
|
|
|
|
|
|
|
|
|
|
if (m_status >= 10 && m_status < 20) {
|
|
|
|
|
m_state = State::Finished;
|
|
|
|
|
} else if (m_status >= 20 && m_status < 30) {
|
|
|
|
|
m_state = State::InBody;
|
|
|
|
|
} else if (m_status >= 30 && m_status < 40) {
|
|
|
|
|
m_state = State::Finished;
|
|
|
|
|
} else if (m_status >= 40 && m_status < 50) {
|
|
|
|
|
m_state = State::Finished;
|
|
|
|
|
} else if (m_status >= 50 && m_status < 60) {
|
|
|
|
|
m_state = State::Finished;
|
|
|
|
|
} else if (m_status >= 60 && m_status < 70) {
|
|
|
|
|
m_state = State::InBody;
|
|
|
|
|
} else {
|
2021-05-31 15:02:30 +01:00
|
|
|
warnln("Job: Expected status between 10 and 69; instead got {}", m_status);
|
2021-08-30 18:12:48 +00:00
|
|
|
return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
|
2020-05-14 18:34:18 +10:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-23 20:42:32 +01:00
|
|
|
VERIFY(m_state == State::InBody || m_state == State::Finished);
|
2020-05-14 18:34:18 +10:00
|
|
|
|
|
|
|
|
read_while_data_available([&] {
|
AK: Rename KB, MB, GB to KiB, MiB, GiB
The SI prefixes "k", "M", "G" mean "10^3", "10^6", "10^9".
The IEC prefixes "Ki", "Mi", "Gi" mean "2^10", "2^20", "2^30".
Let's use the correct name, at least in code.
Only changes the name of the constants, no other behavior change.
2020-08-15 13:55:00 -04:00
|
|
|
auto read_size = 64 * KiB;
|
2020-05-14 18:34:18 +10:00
|
|
|
|
|
|
|
|
auto payload = receive(read_size);
|
2021-05-16 08:47:46 +02:00
|
|
|
if (payload.is_empty()) {
|
2020-05-14 18:34:18 +10:00
|
|
|
if (eof()) {
|
|
|
|
|
finish_up();
|
|
|
|
|
return IterationDecision::Break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (should_fail_on_empty_payload()) {
|
2021-08-30 18:12:48 +00:00
|
|
|
deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
|
2020-05-14 18:34:18 +10:00
|
|
|
return IterationDecision::Break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
m_received_buffers.append(payload);
|
|
|
|
|
m_received_size += payload.size();
|
ProtocolServer: Stream the downloaded data if possible
This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```
Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.
2020-12-26 17:14:12 +03:30
|
|
|
flush_received_buffers();
|
2020-05-14 18:34:18 +10:00
|
|
|
|
2021-08-30 18:12:48 +00:00
|
|
|
deferred_invoke([this] { did_progress({}, m_received_size); });
|
2020-05-14 18:34:18 +10:00
|
|
|
|
|
|
|
|
return IterationDecision::Continue;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (!is_established()) {
|
2021-05-01 21:10:08 +02:00
|
|
|
dbgln_if(JOB_DEBUG, "Connection appears to have closed, finishing up");
|
2020-05-14 18:34:18 +10:00
|
|
|
finish_up();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void Job::finish_up()
|
|
|
|
|
{
|
|
|
|
|
m_state = State::Finished;
|
ProtocolServer: Stream the downloaded data if possible
This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```
Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.
2020-12-26 17:14:12 +03:30
|
|
|
flush_received_buffers();
|
|
|
|
|
if (m_received_size != 0) {
|
2020-12-31 01:42:44 +03:30
|
|
|
// We have to wait for the client to consume all the downloaded data
|
|
|
|
|
// before we can actually call `did_finish`. in a normal flow, this should
|
|
|
|
|
// never be hit since the client is reading as we are writing, unless there
|
|
|
|
|
// are too many concurrent downloads going on.
|
2021-08-30 18:12:48 +00:00
|
|
|
deferred_invoke([this] {
|
2020-12-31 01:42:44 +03:30
|
|
|
finish_up();
|
ProtocolServer: Stream the downloaded data if possible
This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```
Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.
2020-12-26 17:14:12 +03:30
|
|
|
});
|
|
|
|
|
return;
|
2020-05-14 18:34:18 +10:00
|
|
|
}
|
|
|
|
|
|
ProtocolServer: Stream the downloaded data if possible
This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```
Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.
2020-12-26 17:14:12 +03:30
|
|
|
auto response = GeminiResponse::create(m_status, m_meta);
|
2021-08-30 18:12:48 +00:00
|
|
|
deferred_invoke([this, response] {
|
2020-05-14 18:34:18 +10:00
|
|
|
did_finish(move(response));
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|